Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.268 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.018539054 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.019458968 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1201 1202 1203 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.313287378Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.376449732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.380313454Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.380418061Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.38044235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.380470099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"Crucible 0ba9bcaa-f645-4bd8-aad2-da8d1d8bf9a2 has session id: f2c9a90e-5a7c-4a99-a985-8f43b70a58fb","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.381450181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Upstairs opts: Upstairs UUID: 0ba9bcaa-f645-4bd8-aad2-da8d1d8bf9a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.381560087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Crucible stats registered with UUID: 0ba9bcaa-f645-4bd8-aad2-da8d1d8bf9a2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.381584236Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:31.381600385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.382571379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.382663335Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.382688434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383082739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"task":"crutest"} {"msg":"0ba9bcaa-f645-4bd8-aad2-da8d1d8bf9a2 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383192275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383215704Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383244163Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383275382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"2","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.38330748Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383336779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383368248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383391847Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52068)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383809441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48762)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.383919687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59009)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.384016853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.386837636Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"2","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.386876235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.38700357Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388247812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388277911Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388400077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388423866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388443675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388462194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388491273Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388524392Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388544011Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.38856178Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.38858169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388597739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388614488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388630178Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388649417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"downstairs","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388666506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388681646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388698395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388714874Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"client":"2","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388731924Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388747933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"0ba9bcaa-f645-4bd8-aad2-da8d1d8bf9a2 is now active with session: f2c9a90e-5a7c-4a99-a985-8f43b70a58fb","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388766932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388782192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.388797901Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"Activated sub_volume 0ba9bcaa-f645-4bd8-aad2-da8d1d8bf9a2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:31.389248814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.471280128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.471773179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.471858946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.471916453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.471975031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.472036019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.472066867Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.472098686Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.472128665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"f2c9a90e-5a7c-4a99-a985-8f43b70a58fb"} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.504385996Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.54943663Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.550167663Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.55023797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.550258479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.550286168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} {"msg":"Crucible a5c8dd62-fcb1-4377-9228-f3f8d2969b3d has session id: d2d156d1-8604-45b8-b8d5-9ca402b8efcf","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.550837807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Upstairs opts: Upstairs UUID: a5c8dd62-fcb1-4377-9228-f3f8d2969b3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.550893145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Crucible stats registered with UUID: a5c8dd62-fcb1-4377-9228-f3f8d2969b3d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.550923964Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.550953122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551153185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551187844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551205933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551379306Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551398136Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} {"msg":"a5c8dd62-fcb1-4377-9228-f3f8d2969b3d active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551467983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551487182Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"0","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551503941Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"1","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551520771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"2","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.55153717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551561619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551596258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:48792)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551705524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:47350)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551820529Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:42202)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.551864448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.552895119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"1","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553041613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"0","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553193357Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"2","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553534444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.5536397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553842623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553866762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553887291Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.55390538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553923289Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553942709Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553963928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.553982077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554028485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554046015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554062704Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554078543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554094323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"":"downstairs","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554111412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:56:35.554126382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"0","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554148861Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"1","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.55416573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"client":"2","":"downstairs","session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554192129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554208609Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"a5c8dd62-fcb1-4377-9228-f3f8d2969b3d is now active with session: d2d156d1-8604-45b8-b8d5-9ca402b8efcf","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554226348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554241307Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208,"session_id":"d2d156d1-8604-45b8-b8d5-9ca402b8efcf"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554260327Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} {"msg":"Activated sub_volume a5c8dd62-fcb1-4377-9228-f3f8d2969b3d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:35.554274466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 559 len 8192 data: 2 2 02/30 Flush 03/30 Write block 484 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 262 len 40960 05/30 Read block 396 len 28672 06/30 Write block 235 len 20480 data: 2 2 2 2 2 07/30 Flush 08/30 Write block 491 len 36864 data: 3 2 2 2 2 2 2 2 2 09/30 Read block 359 len 4096 10/30 Write block 481 len 20480 data: 2 2 2 3 3 11/30 Read block 309 len 24576 12/30 Read block 33 len 24576 13/30 Read block 549 len 12288 14/30 Flush 15/30 Write block 364 len 4096 data: 2 16/30 Read block 207 len 24576 17/30 Read block 508 len 32768 18/30 Read block 213 len 40960 19/30 Write block 590 len 4096 data: 2 20/30 Write block 15 len 28672 data: 2 2 2 2 2 2 2 21/30 Flush 22/30 Write block 525 len 20480 data: 2 2 2 2 2 23/30 Read block 442 len 28672 24/30 Read block 365 len 36864 25/30 Flush 26/30 Flush 27/30 Write block 145 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 79 len 16384 29/30 Write block 47 len 24576 data: 2 2 2 2 2 2 30/30 Write block 405 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Done Done false 1003 Acked Read 10 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 27 30 Done 27 26 1 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:35.574321642Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 1 5 5 1 F F F <--- 1 020-039 2 2 1 5 5 1 F F F <--- 2 040-059 1 1 1 1 1 1 T T F <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 T T F <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 1 3 3 1 F F F <--- 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 1 5 5 1 F F F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 T T F <--- 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 1 4 4 1 F F T <--- 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 1 6 6 1 F F F <--- 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 2 2 1 5 5 1 F F F <--- Max gen: 2, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 3 3 3 7 7 7 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 3 3 3 7 7 7 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 4 4 4 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 6 6 6 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 2 2 2 5 5 5 F F F Max gen: 3, Max flush: 7 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.115096441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.160536821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.161270473Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.161316651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.16133357Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.161360789Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} {"msg":"Crucible aa05d103-1606-4f89-bfe8-e9c4154b1303 has session id: 9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.161875699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Upstairs opts: Upstairs UUID: aa05d103-1606-4f89-bfe8-e9c4154b1303, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.161932667Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Crucible stats registered with UUID: aa05d103-1606-4f89-bfe8-e9c4154b1303","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.161962866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:40.161988235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162166178Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162204537Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162222806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.16237873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162402999Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} {"msg":"aa05d103-1606-4f89-bfe8-e9c4154b1303 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162467277Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162487276Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"0","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162511275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"1","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162542074Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"2","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162574933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162609902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.16264499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"ds_connection connected from Ok(127.0.0.1:34416)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162677209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"ds_connection connected from Ok(127.0.0.1:47157)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162708868Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"ds_connection connected from Ok(127.0.0.1:44906)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.162742777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.163941011Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"0","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.164079906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"1","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.164606055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.164636414Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.164879555Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"2","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[0]R flush_numbers[0..12]: [5, 5, 7, 1, 1, 1, 1, 7, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165411055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[0]R generation[0..12]: [2, 2, 3, 1, 1, 1, 1, 3, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165442994Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165463763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[1]R flush_numbers[0..12]: [5, 5, 7, 1, 1, 1, 1, 7, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165482002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[1]R generation[0..12]: [2, 2, 3, 1, 1, 1, 1, 3, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165499661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165518981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[2]R flush_numbers[0..12]: [5, 5, 7, 1, 1, 1, 1, 7, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.16553681Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[2]R generation[0..12]: [2, 2, 3, 1, 1, 1, 1, 3, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165554549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165580248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165616777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165641976Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165661645Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165689534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"":"downstairs","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165720413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165751662Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"0","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165771481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"1","":"downstairs","session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.1658099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165846248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165871317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"aa05d103-1606-4f89-bfe8-e9c4154b1303 is now active with session: 9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165889717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165905356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216,"session_id":"9f43a7e9-a7a7-41a7-ad06-e31a4ebe6dac"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165921115Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} {"msg":"Activated sub_volume aa05d103-1606-4f89-bfe8-e9c4154b1303","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:40.165934865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 12288 data: 2 2 2 02/30 Write block 95 len 8192 data: 2 2 03/30 Read block 524 len 20480 04/30 Write block 534 len 24576 data: 2 2 2 2 2 2 05/30 Read block 511 len 40960 06/30 Read block 52 len 32768 07/30 Write block 352 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Read block 594 len 16384 09/30 Write block 483 len 20480 data: 3 4 4 3 3 10/30 Write block 260 len 12288 data: 2 2 2 11/30 Read block 476 len 28672 12/30 Read block 343 len 36864 13/30 Flush 14/30 Write block 120 len 36864 data: 2 2 2 2 2 2 2 2 2 15/30 Read block 217 len 16384 16/30 Write block 60 len 12288 data: 2 2 2 17/30 Read block 0 len 40960 18/30 Read block 186 len 12288 19/30 Flush 20/30 Write block 569 len 4096 data: 2 21/30 Flush 22/30 Write block 222 len 16384 data: 2 2 2 2 23/30 Write block 548 len 28672 data: 2 2 2 2 2 2 2 24/30 Read block 480 len 32768 25/30 Read block 246 len 12288 26/30 Read block 126 len 16384 27/30 Write block 265 len 16384 data: 2 2 2 2 28/30 Write block 80 len 28672 data: 2 2 2 2 2 2 2 29/30 Read block 390 len 40960 30/30 Write block 417 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 5 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:40.185119424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 3 3 3 7 7 7 F F F 3 060-079 4 4 1 9 9 1 F F F <--- 4 080-099 4 4 1 8 8 1 T T T <--- 5 100-119 1 1 1 1 1 1 F F F 6 120-139 4 4 1 9 9 1 F F F <--- 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 T T F <--- 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 1 8 8 1 T T F <--- 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 4 4 1 8 8 1 F F T <--- 17 340-359 4 4 1 8 8 1 F F F <--- 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 3 3 3 7 7 7 T T F <--- 21 420-439 1 1 1 1 1 1 T T F <--- 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 2 8 8 4 F F F <--- 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 2 8 8 6 F F F <--- 27 540-559 2 2 2 2 2 2 T T F <--- 28 560-579 4 4 2 10 10 2 F F F <--- 29 580-599 2 2 2 5 5 5 F F F Max gen: 4, Max flush: 10 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 3 3 3 7 7 7 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 4 4 4 9 9 9 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 5 5 5 11 11 11 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 5 5 5 11 11 11 F F F 21 420-439 5 5 5 11 11 11 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 5 5 5 11 11 11 F F F 28 560-579 4 4 4 10 10 10 F F F 29 580-599 2 2 2 5 5 5 F F F Max gen: 5, Max flush: 11 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.581509092Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.626838546Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.627588517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.627647395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.627668154Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.627703893Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} {"msg":"Crucible 6c54f934-fc94-4a5f-bfd7-76afc3098f47 has session id: 66546925-f587-4a38-ad54-357ea2e1925f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628263981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Upstairs opts: Upstairs UUID: 6c54f934-fc94-4a5f-bfd7-76afc3098f47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628316569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Crucible stats registered with UUID: 6c54f934-fc94-4a5f-bfd7-76afc3098f47","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628337489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:44.628353888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628519282Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.62856636Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628599708Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628747063Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628782652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} {"msg":"6c54f934-fc94-4a5f-bfd7-76afc3098f47 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628808361Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.62883259Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"0","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628864029Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"1","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628896387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"2","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628926906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.628987504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.629029762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52617)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.629064021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35032)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.62910162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57727)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.629138948Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.630369451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"1","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.630469657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"0","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.630519106Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"2","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.6309323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631041395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[0]R flush_numbers[0..12]: [5, 5, 7, 9, 11, 1, 9, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631132102Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[0]R generation[0..12]: [2, 2, 3, 4, 5, 1, 4, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631155831Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.63117579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[1]R flush_numbers[0..12]: [5, 5, 7, 9, 11, 1, 9, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.63119393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[1]R generation[0..12]: [2, 2, 3, 4, 5, 1, 4, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631211399Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631230968Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[2]R flush_numbers[0..12]: [5, 5, 7, 9, 11, 1, 9, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631249078Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[2]R generation[0..12]: [2, 2, 3, 4, 5, 1, 4, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631270237Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631289906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631306005Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631322775Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631345834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631361833Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"":"downstairs","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631378593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631393612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"0","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631410101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"1","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631426681Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"client":"2","":"downstairs","session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.63144364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.63145958Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"6c54f934-fc94-4a5f-bfd7-76afc3098f47 is now active with session: 66546925-f587-4a38-ad54-357ea2e1925f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631477209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631492348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224,"session_id":"66546925-f587-4a38-ad54-357ea2e1925f"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:44.631508268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} {"msg":"Activated sub_volume 6c54f934-fc94-4a5f-bfd7-76afc3098f47","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-14T23:56:44.631521997Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 488 len 20480 data: 3 3 3 4 3 02/30 Read block 100 len 8192 03/30 Read block 432 len 40960 04/30 Flush 05/30 Write block 588 len 12288 data: 2 2 3 06/30 Read block 321 len 32768 07/30 Write block 337 len 24576 data: 2 2 2 2 2 2 08/30 Read block 194 len 8192 09/30 Read block 147 len 4096 10/30 Write block 417 len 32768 data: 3 3 3 3 3 3 2 2 11/30 Read block 199 len 4096 12/30 Flush 13/30 Write block 300 len 12288 data: 2 2 2 14/30 Read block 337 len 32768 15/30 Read block 438 len 16384 16/30 Write block 21 len 40960 data: 3 2 2 2 2 2 2 2 2 2 17/30 Write block 1 len 4096 data: 2 18/30 Read block 16 len 24576 19/30 Read block 436 len 8192 20/30 Read block 556 len 32768 21/30 Write block 338 len 40960 data: 3 3 3 3 3 2 2 2 2 2 22/30 Read block 470 len 36864 23/30 Write block 513 len 32768 data: 2 2 2 2 2 2 2 2 24/30 Read block 448 len 40960 25/30 Flush 26/30 Write block 121 len 32768 data: 3 3 3 3 3 3 3 3 27/30 Read block 235 len 28672 28/30 Write block 438 len 16384 data: 2 2 2 2 29/30 Read block 210 len 32768 30/30 Read block 113 len 12288 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 10 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:44.652250718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 2 14 14 5 F F F <--- 1 020-039 6 6 2 14 14 5 F F F <--- 2 040-059 3 3 3 7 7 7 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 4 4 4 9 9 9 T T F <--- 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 5 5 5 11 11 11 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 6 6 1 14 14 1 F F F <--- 16 320-339 6 6 4 14 14 8 F F F <--- 17 340-359 6 6 4 14 14 8 F F F <--- 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 6 6 5 13 13 11 F F F <--- 21 420-439 6 6 5 13 13 11 T T F <--- 22 440-459 1 1 1 1 1 1 T T F <--- 23 460-479 1 1 1 1 1 1 F F F 24 480-499 6 6 4 12 12 8 F F T <--- 25 500-519 6 6 1 14 14 1 F F F <--- 26 520-539 6 6 4 14 14 8 F F F <--- 27 540-559 5 5 5 11 11 11 F F F 28 560-579 4 4 4 10 10 10 F F F 29 580-599 6 6 2 13 13 5 F F F <--- Max gen: 6, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 14 14 14 F F F 1 020-039 6 6 6 14 14 14 F F F 2 040-059 3 3 3 7 7 7 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 7 7 7 15 15 15 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 5 5 5 11 11 11 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 6 6 6 14 14 14 F F F 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 7 7 7 15 15 15 F F F 22 440-459 7 7 7 15 15 15 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 6 6 6 12 12 12 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 6 6 6 14 14 14 F F F 27 540-559 5 5 5 11 11 11 F F F 28 560-579 4 4 4 10 10 10 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 7, Max flush: 15 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.048104846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:49.09340691Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.093915011Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.126574207Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.127559729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.127615967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.127634626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.127660145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} {"msg":"Crucible 4993847f-e780-4d2f-a924-2723491cdd75 has session id: 8037b0ab-53a8-4cc0-8b13-74cba8d4cabb","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128154647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Upstairs opts: Upstairs UUID: 4993847f-e780-4d2f-a924-2723491cdd75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128218304Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Crucible stats registered with UUID: 4993847f-e780-4d2f-a924-2723491cdd75","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128249933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:49.128277182Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128468855Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128512733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128533422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128661167Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128682526Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} {"msg":"4993847f-e780-4d2f-a924-2723491cdd75 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128743844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128773633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"0","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128792892Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"1","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128810022Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"2","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128826711Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.12885443Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128886129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41769)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.128959856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"ds_connection connected from Ok(127.0.0.1:40623)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.129011384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45148)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.129045593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.130313404Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"2","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.13043825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"1","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.130463829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"0","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.130867203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131020668Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 7, 9, 11, 1, 15, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131131673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[0]R generation[0..12]: [6, 6, 3, 4, 5, 1, 7, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131155372Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131175622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 7, 9, 11, 1, 15, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131193971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[1]R generation[0..12]: [6, 6, 3, 4, 5, 1, 7, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.13121174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131231229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 7, 9, 11, 1, 15, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131249279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[2]R generation[0..12]: [6, 6, 3, 4, 5, 1, 7, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131266968Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131286267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131302437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131319326Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131341585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131357905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"":"downstairs","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131374814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131393823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"0","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131411273Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"1","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131427982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"client":"2","":"downstairs","session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131445291Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131461321Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"4993847f-e780-4d2f-a924-2723491cdd75 is now active with session: 8037b0ab-53a8-4cc0-8b13-74cba8d4cabb","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.13147905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.13149429Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232,"session_id":"8037b0ab-53a8-4cc0-8b13-74cba8d4cabb"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131510109Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} {"msg":"Activated sub_volume 4993847f-e780-4d2f-a924-2723491cdd75","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:49.131525918Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 250 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 284 len 36864 03/30 Write block 569 len 40960 data: 3 2 2 2 2 2 2 2 2 2 04/30 Read block 224 len 4096 05/30 Read block 415 len 36864 06/30 Read block 586 len 12288 07/30 Write block 470 len 4096 data: 2 08/30 Read block 5 len 40960 09/30 Read block 292 len 16384 10/30 Write block 69 len 24576 data: 2 2 2 2 2 2 11/30 Read block 383 len 36864 12/30 Write block 51 len 20480 data: 3 3 2 2 2 13/30 Read block 361 len 28672 14/30 Read block 472 len 28672 15/30 Write block 22 len 24576 data: 3 3 3 3 3 3 16/30 Read block 449 len 40960 17/30 Read block 538 len 16384 18/30 Read block 432 len 8192 19/30 Flush 20/30 Write block 278 len 32768 data: 2 2 2 2 2 2 2 2 21/30 Read block 440 len 12288 22/30 Write block 12 len 24576 data: 2 2 2 3 3 3 23/30 Read block 234 len 8192 24/30 Write block 309 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Read block 250 len 28672 26/30 Read block 225 len 36864 27/30 Read block 414 len 40960 28/30 Read block 258 len 32768 29/30 Read block 542 len 24576 30/30 Write block 293 len 40960 data: 2 2 2 2 2 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:49.154224524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 14 14 14 F T T <--- 1 020-039 6 8 8 14 16 16 F F F <--- 2 040-059 3 8 8 7 16 16 F F F <--- 3 060-079 4 8 8 9 16 16 F F F <--- 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 7 7 7 15 15 15 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 1 8 8 1 16 16 T F F <--- 13 260-279 5 5 5 11 11 11 F T T <--- 14 280-299 1 1 1 1 1 1 F T T <--- 15 300-319 6 6 6 14 14 14 F T T <--- 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 7 7 7 15 15 15 F F F 22 440-459 7 7 7 15 15 15 F F F 23 460-479 1 8 8 1 16 16 F F F <--- 24 480-499 6 6 6 12 12 12 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 6 6 6 14 14 14 F F F 27 540-559 5 5 5 11 11 11 F F F 28 560-579 4 8 8 10 16 16 F F F <--- 29 580-599 6 6 6 13 13 13 F F F Max gen: 8, Max flush: 16 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 17 17 17 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 8 8 8 16 16 16 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 7 7 7 15 15 15 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 8 8 8 16 16 16 F F F 13 260-279 9 9 9 17 17 17 F F F 14 280-299 9 9 9 17 17 17 F F F 15 300-319 9 9 9 17 17 17 F F F 16 320-339 6 6 6 14 14 14 F F F 17 340-359 6 6 6 14 14 14 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 7 7 7 15 15 15 F F F 22 440-459 7 7 7 15 15 15 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 6 6 6 12 12 12 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 6 6 6 14 14 14 F F F 27 540-559 5 5 5 11 11 11 F F F 28 560-579 8 8 8 16 16 16 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 9, Max flush: 17 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.603061964Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.648090439Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.64883766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.648894908Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.648912628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.648934527Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} {"msg":"Crucible bd885663-59a5-41ab-9525-04231d250337 has session id: cc1fddf1-b974-4e98-8c99-839a53b9758f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649474466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Upstairs opts: Upstairs UUID: bd885663-59a5-41ab-9525-04231d250337, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649526804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Crucible stats registered with UUID: bd885663-59a5-41ab-9525-04231d250337","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649547323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:53.649563133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649780644Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649831693Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649862281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.649988807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650019275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} {"msg":"bd885663-59a5-41ab-9525-04231d250337 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650086753Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650108282Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"0","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650127671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"1","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.65015567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"2","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650185829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650214978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650243247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46866)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650288745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55833)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650323894Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52777)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.650460438Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.651778078Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"0","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.651879024Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"1","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.652356466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.652422744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.652624786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"2","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[0]R flush_numbers[0..12]: [17, 16, 16, 16, 11, 1, 15, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653181315Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[0]R generation[0..12]: [9, 8, 8, 8, 5, 1, 7, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653205784Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653225893Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[1]R flush_numbers[0..12]: [17, 16, 16, 16, 11, 1, 15, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653244452Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[1]R generation[0..12]: [9, 8, 8, 8, 5, 1, 7, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653262352Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653281781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[2]R flush_numbers[0..12]: [17, 16, 16, 16, 11, 1, 15, 7, 1, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.6533001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[2]R generation[0..12]: [9, 8, 8, 8, 5, 1, 7, 3, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653322769Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653342479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653358718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653375518Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Next flush: 18Wait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653397807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653413786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"":"downstairs","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653430535Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653445975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"0","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653463894Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"1","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653480763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"client":"2","":"downstairs","session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653498063Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653514022Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"bd885663-59a5-41ab-9525-04231d250337 is now active with session: cc1fddf1-b974-4e98-8c99-839a53b9758f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653531652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.653546671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240,"session_id":"cc1fddf1-b974-4e98-8c99-839a53b9758f"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.65356266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} {"msg":"Activated sub_volume bd885663-59a5-41ab-9525-04231d250337","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:53.65357646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 452 len 20480 data: 2 2 2 2 2 02/30 Write block 59 len 24576 data: 2 3 3 3 2 2 03/30 Read block 162 len 40960 04/30 Read block 549 len 32768 05/30 Write block 12 len 4096 data: 3 06/30 Read block 13 len 40960 07/30 Read block 430 len 28672 08/30 Write block 421 len 16384 data: 4 4 3 3 09/30 Read block 188 len 20480 10/30 Write block 253 len 24576 data: 3 3 3 3 3 3 11/30 Read block 151 len 32768 12/30 Read block 315 len 40960 13/30 Read block 258 len 12288 14/30 Write block 28 len 8192 data: 3 3 15/30 Flush 16/30 Flush 17/30 Flush 18/30 Write block 70 len 20480 data: 3 3 3 3 3 19/30 Write block 182 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Write block 536 len 24576 data: 3 3 3 3 2 2 21/30 Read block 259 len 8192 22/30 Read block 383 len 32768 23/30 Read block 217 len 20480 24/30 Flush 25/30 Write block 73 len 4096 data: 4 26/30 Read block 236 len 24576 27/30 Write block 334 len 28672 data: 2 2 2 3 4 4 4 28/30 Read block 581 len 36864 29/30 Write block 40 len 40960 data: 2 2 2 2 2 2 2 3 3 3 30/30 Write block 200 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 10 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:53.672312716Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 9 18 18 17 F F F <--- 1 020-039 10 10 8 18 18 16 F F F <--- 2 040-059 10 10 8 18 18 16 T T T <--- 3 060-079 10 10 8 21 21 16 T T T <--- 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 7 7 7 15 15 15 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 1 21 21 1 F F F <--- 10 200-219 1 1 1 1 1 1 T T F <--- 11 220-239 5 5 5 11 11 11 F F F 12 240-259 10 10 8 18 18 16 F F F <--- 13 260-279 9 9 9 17 17 17 F F F 14 280-299 9 9 9 17 17 17 F F F 15 300-319 9 9 9 17 17 17 F F F 16 320-339 6 6 6 14 14 14 T T F <--- 17 340-359 6 6 6 14 14 14 T T F <--- 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 10 10 7 18 18 15 F F F <--- 22 440-459 10 10 7 18 18 15 F F T <--- 23 460-479 8 8 8 16 16 16 F F F 24 480-499 6 6 6 12 12 12 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 6 21 21 14 F F F <--- 27 540-559 10 10 5 21 21 11 F F F <--- 28 560-579 8 8 8 16 16 16 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 10, Max flush: 21 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 18 18 18 F F F 1 020-039 10 10 10 18 18 18 F F F 2 040-059 11 11 11 22 22 22 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 5 5 5 11 11 11 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 7 7 7 15 15 15 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 10 21 21 21 F F F 10 200-219 11 11 11 22 22 22 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 10 10 10 18 18 18 F F F 13 260-279 9 9 9 17 17 17 F F F 14 280-299 9 9 9 17 17 17 F F F 15 300-319 9 9 9 17 17 17 F F F 16 320-339 11 11 11 22 22 22 F F F 17 340-359 11 11 11 22 22 22 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 10 10 10 18 18 18 F F F 22 440-459 10 10 10 18 18 18 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 6 6 6 12 12 12 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 8 8 8 16 16 16 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 11, Max flush: 22 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.077301127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.121978365Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.122715587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.122766285Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.122793434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.122830913Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} {"msg":"Crucible 6355997a-8efb-41bb-b7b4-188593eedd44 has session id: d1bbe574-8363-431d-a760-1b7f3358887a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.123316324Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Upstairs opts: Upstairs UUID: 6355997a-8efb-41bb-b7b4-188593eedd44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.123395421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Crucible stats registered with UUID: 6355997a-8efb-41bb-b7b4-188593eedd44","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.12342149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:58.123455809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.123717549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.123759427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.123779597Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.123982079Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124005128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} {"msg":"6355997a-8efb-41bb-b7b4-188593eedd44 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124064346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124084725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"0","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124110404Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"1","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124141203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"2","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124174042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.12420674Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124258618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62893)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124293027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61740)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124325936Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60433)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.124382154Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.125674684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"1","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.125710143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"0","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.125745032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"2","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126278772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126376238Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[0]R flush_numbers[0..12]: [18, 18, 22, 22, 11, 1, 15, 7, 1, 21, 22, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126505033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[0]R generation[0..12]: [10, 10, 11, 11, 5, 1, 7, 3, 1, 10, 11, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126542591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126565101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[1]R flush_numbers[0..12]: [18, 18, 22, 22, 11, 1, 15, 7, 1, 21, 22, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.12658385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[1]R generation[0..12]: [10, 10, 11, 11, 5, 1, 7, 3, 1, 10, 11, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126602049Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126621568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[2]R flush_numbers[0..12]: [18, 18, 22, 22, 11, 1, 15, 7, 1, 21, 22, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126639778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[2]R generation[0..12]: [10, 10, 11, 11, 5, 1, 7, 3, 1, 10, 11, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126657757Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126677666Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126694186Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126710935Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Next flush: 23","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:56:58.126726774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126750394Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"":"downstairs","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126767453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126782622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"0","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126801781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"1","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126819111Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"client":"2","":"downstairs","session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.1268362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126852379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"6355997a-8efb-41bb-b7b4-188593eedd44 is now active with session: d1bbe574-8363-431d-a760-1b7f3358887a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126870339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126885628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248,"session_id":"d1bbe574-8363-431d-a760-1b7f3358887a"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126901528Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} {"msg":"Activated sub_volume 6355997a-8efb-41bb-b7b4-188593eedd44","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:56:58.126915147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 130 len 24576 data: 2 2 2 2 2 2 03/30 Flush 04/30 Write block 292 len 40960 data: 2 3 3 3 3 3 3 3 4 4 05/30 Read block 441 len 20480 06/30 Flush 07/30 Write block 13 len 40960 data: 3 3 4 4 4 3 3 3 4 4 08/30 Read block 67 len 32768 09/30 Write block 365 len 28672 data: 2 2 2 2 2 2 2 10/30 Read block 374 len 24576 11/30 Write block 423 len 8192 data: 4 4 12/30 Read block 88 len 4096 13/30 Flush 14/30 Flush 15/30 Write block 97 len 8192 data: 2 2 16/30 Write block 488 len 40960 data: 4 4 4 5 4 3 3 3 3 3 17/30 Read block 137 len 4096 18/30 Flush 19/30 Write block 422 len 32768 data: 5 5 5 2 2 2 2 2 20/30 Write block 24 len 36864 data: 4 4 4 4 4 4 3 2 2 21/30 Read block 296 len 24576 22/30 Write block 253 len 40960 data: 4 4 4 4 4 4 3 3 3 3 23/30 Read block 151 len 40960 24/30 Read block 272 len 28672 25/30 Read block 490 len 20480 26/30 Flush 27/30 Write block 390 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Read block 109 len 8192 29/30 Write block 182 len 20480 data: 3 3 3 3 3 30/30 Read block 405 len 24576 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 27 0 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1002)) Some(JobId(1025)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:56:58.148237905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 10 12 26 18 26 F F F <--- 1 020-039 12 10 12 29 18 29 F F F <--- 2 040-059 11 11 11 22 22 22 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 12 5 12 28 11 28 F F F <--- 5 100-119 1 1 1 1 1 1 F F F 6 120-139 12 12 12 24 24 24 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 10 21 21 21 T F T <--- 10 200-219 11 11 11 22 22 22 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 12 10 12 29 18 29 F F F <--- 13 260-279 12 9 12 29 17 29 F F F <--- 14 280-299 12 9 12 25 17 25 F F F <--- 15 300-319 12 9 12 25 17 25 F F F <--- 16 320-339 11 11 11 22 22 22 F F F 17 340-359 11 11 11 22 22 22 F F F 18 360-379 12 2 12 26 5 26 F F F <--- 19 380-399 1 1 1 1 1 1 T F T <--- 20 400-419 6 6 6 13 13 13 F F F 21 420-439 12 10 12 29 18 29 F F F <--- 22 440-459 10 10 10 18 18 18 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 6 12 28 12 28 F F F <--- 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 8 8 8 16 16 16 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 12, Max flush: 29 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 26 26 26 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 11 11 11 22 22 22 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 12 12 12 24 24 24 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 13 13 13 30 30 30 F F F 10 200-219 11 11 11 22 22 22 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 12 12 12 29 29 29 F F F 13 260-279 12 12 12 29 29 29 F F F 14 280-299 12 12 12 25 25 25 F F F 15 300-319 12 12 12 25 25 25 F F F 16 320-339 11 11 11 22 22 22 F F F 17 340-359 11 11 11 22 22 22 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 12 12 12 29 29 29 F F F 22 440-459 10 10 10 18 18 18 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 8 8 8 16 16 16 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 13, Max flush: 30 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.53919723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.583638137Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.584384679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.584432487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.584448467Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.584470386Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} {"msg":"Crucible 996ad105-becd-400d-9f35-3f67a6471e06 has session id: 2b7e9bca-d760-47a2-97c9-db9bde23b6c4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.58487146Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Upstairs opts: Upstairs UUID: 996ad105-becd-400d-9f35-3f67a6471e06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.584978566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Crucible stats registered with UUID: 996ad105-becd-400d-9f35-3f67a6471e06","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585008575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:02.585025734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585213347Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585264305Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585293684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.58540601Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585436279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} {"msg":"996ad105-becd-400d-9f35-3f67a6471e06 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585474508Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585500627Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"0","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585532735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"1","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585565264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"2","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585597043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585631531Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.58566355Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47599)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585693509Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54193)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585723288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46798)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.585759766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.586922422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"0","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.586970371Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"2","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.587596127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.587635295Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.588839469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"1","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[0]R flush_numbers[0..12]: [26, 29, 22, 22, 28, 1, 24, 7, 1, 30, 22, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589382099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[0]R generation[0..12]: [12, 12, 11, 11, 12, 1, 12, 3, 1, 13, 11, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589406568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589438966Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[1]R flush_numbers[0..12]: [26, 29, 22, 22, 28, 1, 24, 7, 1, 30, 22, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589459286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[1]R generation[0..12]: [12, 12, 11, 11, 12, 1, 12, 3, 1, 13, 11, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589477615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589496944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[2]R flush_numbers[0..12]: [26, 29, 22, 22, 28, 1, 24, 7, 1, 30, 22, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589515223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[2]R generation[0..12]: [12, 12, 11, 11, 12, 1, 12, 3, 1, 13, 11, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589533223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589555892Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589572401Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589589051Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.58960471Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589620679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"":"downstairs","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589645689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589661118Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"0","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589677887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"1","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589694517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"client":"2","":"downstairs","session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589712446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589728535Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"996ad105-becd-400d-9f35-3f67a6471e06 is now active with session: 2b7e9bca-d760-47a2-97c9-db9bde23b6c4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589746405Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589761444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256,"session_id":"2b7e9bca-d760-47a2-97c9-db9bde23b6c4"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589777464Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} {"msg":"Activated sub_volume 996ad105-becd-400d-9f35-3f67a6471e06","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:02.589791123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 575 len 40960 data: 3 3 3 3 2 2 2 2 2 2 02/30 Write block 309 len 16384 data: 3 3 3 3 03/30 Write block 197 len 16384 data: 2 2 2 3 04/30 Read block 424 len 24576 05/30 Read block 299 len 28672 06/30 Read block 151 len 40960 07/30 Read block 27 len 20480 08/30 Read block 171 len 8192 09/30 Read block 352 len 28672 10/30 Read block 404 len 32768 11/30 Write block 179 len 4096 data: 2 12/30 Read block 83 len 4096 13/30 Read block 381 len 16384 14/30 Read block 114 len 40960 15/30 Read block 574 len 4096 16/30 Write block 439 len 32768 data: 3 3 3 2 2 2 2 2 17/30 Write block 439 len 40960 data: 4 4 4 3 3 3 3 3 2 2 18/30 Read block 96 len 40960 19/30 Read block 580 len 36864 20/30 Read block 278 len 4096 21/30 Read block 59 len 8192 22/30 Flush 23/30 Write block 587 len 4096 data: 2 24/30 Write block 594 len 24576 data: 2 2 2 2 2 2 25/30 Read block 217 len 32768 26/30 Write block 204 len 36864 data: 3 3 2 2 2 2 2 2 2 27/30 Read block 410 len 36864 28/30 Read block 523 len 40960 29/30 Read block 465 len 24576 30/30 Write block 561 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 21 1 23 Done 29 9 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:02.609764882Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 26 26 26 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 11 11 11 22 22 22 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 12 12 12 24 24 24 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 14 1 14 31 1 31 F F F <--- 9 180-199 14 13 14 31 30 31 F T F <--- 10 200-219 14 11 14 31 22 31 T T T <--- 11 220-239 5 5 5 11 11 11 F F F 12 240-259 12 12 12 29 29 29 F F F 13 260-279 12 12 12 29 29 29 F F F 14 280-299 12 12 12 25 25 25 F F F 15 300-319 14 12 14 31 25 31 F T F <--- 16 320-339 11 11 11 22 22 22 F F F 17 340-359 11 11 11 22 22 22 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 14 12 14 31 29 31 F F F <--- 22 440-459 14 10 14 31 18 31 F F F <--- 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 14 8 14 31 16 31 T T T <--- 29 580-599 14 6 14 31 13 31 T T T <--- Max gen: 14, Max flush: 31 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 26 26 26 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 11 11 11 22 22 22 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 12 12 12 24 24 24 F F F 7 140-159 3 3 3 7 7 7 F F F 8 160-179 14 14 14 31 31 31 F F F 9 180-199 14 14 14 31 31 31 F F F 10 200-219 15 15 15 32 32 32 F F F 11 220-239 5 5 5 11 11 11 F F F 12 240-259 12 12 12 29 29 29 F F F 13 260-279 12 12 12 29 29 29 F F F 14 280-299 12 12 12 25 25 25 F F F 15 300-319 14 14 14 31 31 31 F F F 16 320-339 11 11 11 22 22 22 F F F 17 340-359 11 11 11 22 22 22 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 14 14 14 31 31 31 F F F 22 440-459 14 14 14 31 31 31 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 15 15 15 32 32 32 F F F 29 580-599 15 15 15 32 32 32 F F F Max gen: 15, Max flush: 32 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:06.990500426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:07.034969562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.035391746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.067274252Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068086401Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068147929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068168758Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068196147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} {"msg":"Crucible e6d6f697-b765-4a38-800b-28215a98141e has session id: 86dd74a5-91ab-49bc-a10e-dc23b31f4f30","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068750426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Upstairs opts: Upstairs UUID: e6d6f697-b765-4a38-800b-28215a98141e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068800574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Crucible stats registered with UUID: e6d6f697-b765-4a38-800b-28215a98141e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.068823783Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:07.068843412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069027645Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069077673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069096932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069286045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069311154Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} {"msg":"e6d6f697-b765-4a38-800b-28215a98141e active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069382051Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069403851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"0","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.06942147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"1","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069443779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"2","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069461059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069529806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069574454Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"ds_connection connected from Ok(127.0.0.1:53920)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069609283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"ds_connection connected from Ok(127.0.0.1:49503)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069642812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"ds_connection connected from Ok(127.0.0.1:54505)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.069775836Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.070872745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"1","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.07098419Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"0","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071046788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"2","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.07152427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071623846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[0]R flush_numbers[0..12]: [26, 29, 22, 22, 28, 1, 24, 7, 31, 31, 32, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071660235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[0]R generation[0..12]: [12, 12, 11, 11, 12, 1, 12, 3, 14, 14, 15, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071680584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071700573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[1]R flush_numbers[0..12]: [26, 29, 22, 22, 28, 1, 24, 7, 31, 31, 32, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071718973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[1]R generation[0..12]: [12, 12, 11, 11, 12, 1, 12, 3, 14, 14, 15, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071737262Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071756821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[2]R flush_numbers[0..12]: [26, 29, 22, 22, 28, 1, 24, 7, 31, 31, 32, 11]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.07177514Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[2]R generation[0..12]: [12, 12, 11, 11, 12, 1, 12, 3, 14, 14, 15, 5]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.07179326Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071812639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071828818Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071845678Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071861567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071877786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"":"downstairs","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:57:07.071894616Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071914985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"0","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071933314Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"1","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071950264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"client":"2","":"downstairs","session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071967013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.071983072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"e6d6f697-b765-4a38-800b-28215a98141e is now active with session: 86dd74a5-91ab-49bc-a10e-dc23b31f4f30","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.072000932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.072016101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264,"session_id":"86dd74a5-91ab-49bc-a10e-dc23b31f4f30"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.07203199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} {"msg":"Activated sub_volume e6d6f697-b765-4a38-800b-28215a98141e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:07.07204598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 244 len 36864 data: 2 2 2 2 2 2 3 3 3 02/30 Write block 5 len 20480 data: 2 2 2 2 2 03/30 Read block 147 len 32768 04/30 Write block 301 len 20480 data: 5 4 2 2 2 05/30 Flush 06/30 Write block 229 len 36864 data: 2 2 2 2 2 2 3 3 3 07/30 Read block 211 len 24576 08/30 Flush 09/30 Write block 136 len 28672 data: 2 2 2 2 2 2 2 10/30 Write block 268 len 8192 data: 3 2 11/30 Flush 12/30 Write block 161 len 4096 data: 2 13/30 Read block 248 len 16384 14/30 Write block 135 len 16384 data: 3 3 3 3 15/30 Read block 44 len 32768 16/30 Read block 241 len 24576 17/30 Write block 344 len 40960 data: 3 3 3 3 2 2 2 2 3 3 18/30 Read block 79 len 28672 19/30 Write block 309 len 28672 data: 4 4 4 4 3 3 3 20/30 Write block 405 len 16384 data: 3 3 3 3 21/30 Write block 435 len 24576 data: 2 2 2 3 5 5 22/30 Read block 303 len 20480 23/30 Read block 485 len 28672 24/30 Write block 403 len 12288 data: 2 2 4 25/30 Write block 76 len 12288 data: 2 2 2 26/30 Read block 567 len 4096 27/30 Write block 198 len 16384 data: 3 3 4 3 28/30 Write block 127 len 24576 data: 4 4 2 3 3 3 29/30 Read block 99 len 4096 30/30 Read block 388 len 4096 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 11 0 0 11 Done 8 19 19 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:07.090991988Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 33 33 33 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 11 11 11 22 22 22 F F F 3 060-079 11 11 11 22 22 22 F T T <--- 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 16 16 16 35 35 35 T T T 7 140-159 16 16 16 35 35 35 F F F 8 160-179 14 14 14 31 31 31 T T T 9 180-199 14 14 14 31 31 31 F T T <--- 10 200-219 15 15 15 32 32 32 F T T <--- 11 220-239 16 16 16 34 34 34 F F F 12 240-259 16 16 16 33 33 33 F F F 13 260-279 16 16 16 35 35 35 F F F 14 280-299 12 12 12 25 25 25 F F F 15 300-319 16 16 16 33 33 33 T T T 16 320-339 11 11 11 22 22 22 F F F 17 340-359 11 11 11 22 22 22 T T T 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 6 6 6 13 13 13 F T T <--- 21 420-439 14 14 14 31 31 31 F T T <--- 22 440-459 14 14 14 31 31 31 F T T <--- 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 15 15 15 32 32 32 F F F 29 580-599 15 15 15 32 32 32 F F F Max gen: 16, Max flush: 35 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 8 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 129a130,132 > 4, > 4, > 2, 132c135 < 1, --- > 3, 135,137d137 < 2, < 2, < 2, diff found after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 33 33 33 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 11 11 11 22 22 22 F F F 3 060-079 17 17 17 36 36 36 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 17 17 17 36 36 36 F F F 7 140-159 16 16 16 35 35 35 F F F 8 160-179 17 17 17 36 36 36 F F F 9 180-199 17 17 17 36 36 36 F F F 10 200-219 17 17 17 36 36 36 F F F 11 220-239 16 16 16 34 34 34 F F F 12 240-259 16 16 16 33 33 33 F F F 13 260-279 16 16 16 35 35 35 F F F 14 280-299 12 12 12 25 25 25 F F F 15 300-319 17 17 17 36 36 36 F F F 16 320-339 11 11 11 22 22 22 F F F 17 340-359 17 17 17 36 36 36 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 17 17 17 36 36 36 F F F 21 420-439 17 17 17 36 36 36 F F F 22 440-459 17 17 17 36 36 36 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 6 6 6 14 14 14 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 15 15 15 32 32 32 F F F 29 580-599 15 15 15 32 32 32 F F F Max gen: 17, Max flush: 36 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.534654426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.580034518Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.580808818Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.580857726Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.580873806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.580895625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} {"msg":"Crucible fcb3a54b-4a06-4d70-99b3-dfdfa7bb594e has session id: c476e647-7364-494b-9f8f-f7911ff57f37","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.58153009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Upstairs opts: Upstairs UUID: fcb3a54b-4a06-4d70-99b3-dfdfa7bb594e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.581582328Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Crucible stats registered with UUID: fcb3a54b-4a06-4d70-99b3-dfdfa7bb594e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.581607187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:11.581631897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.581870948Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.581920466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.581952474Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582100369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582133427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} {"msg":"fcb3a54b-4a06-4d70-99b3-dfdfa7bb594e active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582159286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582187036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"0","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582218494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"1","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582251183Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"2","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582282542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582314171Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582347869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"ds_connection connected from Ok(127.0.0.1:63638)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582382488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"ds_connection connected from Ok(127.0.0.1:48294)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582432026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"ds_connection connected from Ok(127.0.0.1:43532)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.582528623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.583684258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"2","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.583731626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"0","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.583810204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"1","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584321354Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584351713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[0]R flush_numbers[0..12]: [33, 29, 22, 36, 28, 1, 36, 35, 36, 36, 36, 34]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584375832Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[0]R generation[0..12]: [16, 12, 11, 17, 12, 1, 17, 16, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584395251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.58441532Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[1]R flush_numbers[0..12]: [33, 29, 22, 36, 28, 1, 36, 35, 36, 36, 36, 34]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.58443418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[1]R generation[0..12]: [16, 12, 11, 17, 12, 1, 17, 16, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584452319Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584471698Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[2]R flush_numbers[0..12]: [33, 29, 22, 36, 28, 1, 36, 35, 36, 36, 36, 34]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584489968Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[2]R generation[0..12]: [16, 12, 11, 17, 12, 1, 17, 16, 17, 17, 17, 16]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584508077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584527476Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584548466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584565845Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-14T23:57:11.584581504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584604423Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"":"downstairs","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584621183Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584636282Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"0","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584653002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"1","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584669751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"client":"2","":"downstairs","session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.58468653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.58470255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"fcb3a54b-4a06-4d70-99b3-dfdfa7bb594e is now active with session: c476e647-7364-494b-9f8f-f7911ff57f37","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584720299Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584735448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275,"session_id":"c476e647-7364-494b-9f8f-f7911ff57f37"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584751338Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} {"msg":"Activated sub_volume fcb3a54b-4a06-4d70-99b3-dfdfa7bb594e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:11.584765267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 12288 data: 2 2 2 02/30 Read block 497 len 36864 03/30 Write block 583 len 40960 data: 3 3 2 2 3 3 3 4 2 2 04/30 Write block 49 len 24576 data: 4 3 4 4 3 3 05/30 Flush 06/30 Write block 402 len 28672 data: 2 3 3 5 4 4 4 07/30 Read block 457 len 20480 08/30 Read block 555 len 40960 09/30 Read block 159 len 36864 10/30 Write block 196 len 20480 data: 2 3 4 4 5 11/30 Read block 387 len 4096 12/30 Read block 340 len 28672 13/30 Write block 563 len 36864 data: 3 3 2 2 2 2 4 3 3 14/30 Read block 250 len 4096 15/30 Read block 362 len 20480 16/30 Flush 17/30 Write block 444 len 4096 data: 4 18/30 Write block 515 len 24576 data: 3 3 3 3 3 3 19/30 Read block 206 len 40960 20/30 Write block 263 len 16384 data: 2 2 3 3 21/30 Read block 524 len 12288 22/30 Read block 354 len 40960 23/30 Write block 319 len 20480 data: 2 2 3 3 3 24/30 Write block 8 len 36864 data: 3 3 2 2 4 4 4 5 5 25/30 Read block 42 len 40960 26/30 Write block 541 len 4096 data: 3 27/30 Read block 428 len 24576 28/30 Read block 338 len 32768 29/30 Write block 485 len 36864 data: 5 4 4 5 5 5 6 5 4 30/30 Read block 479 len 12288 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:11.605053794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1275} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 33 33 33 T F T <--- 1 020-039 12 12 12 29 29 29 F F F 2 040-059 18 11 18 37 22 37 F F F <--- 3 060-079 17 17 17 36 36 36 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 17 17 17 36 36 36 F F F 7 140-159 16 16 16 35 35 35 F F F 8 160-179 17 17 17 36 36 36 F F F 9 180-199 18 17 18 38 36 38 F F F <--- 10 200-219 18 17 18 38 36 38 F F F <--- 11 220-239 16 16 16 34 34 34 F F F 12 240-259 16 16 16 33 33 33 F F F 13 260-279 16 16 16 35 35 35 T F T <--- 14 280-299 12 12 12 25 25 25 F F F 15 300-319 17 17 17 36 36 36 T F T <--- 16 320-339 11 11 11 22 22 22 T F T <--- 17 340-359 17 17 17 36 36 36 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 17 18 38 36 38 F F F <--- 21 420-439 17 17 17 36 36 36 F F F 22 440-459 17 17 17 36 36 36 T F T <--- 23 460-479 8 8 8 16 16 16 F F F 24 480-499 12 12 12 28 28 28 T F T <--- 25 500-519 18 6 18 37 14 37 T F T <--- 26 520-539 10 10 10 21 21 21 T F T <--- 27 540-559 10 10 10 21 21 21 T F T <--- 28 560-579 18 15 18 38 32 38 F F F <--- 29 580-599 18 15 18 37 32 37 F F F <--- Max gen: 18, Max flush: 38 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 39 39 39 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 18 18 18 37 37 37 F F F 3 060-079 17 17 17 36 36 36 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 17 17 17 36 36 36 F F F 7 140-159 16 16 16 35 35 35 F F F 8 160-179 17 17 17 36 36 36 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 18 18 18 38 38 38 F F F 11 220-239 16 16 16 34 34 34 F F F 12 240-259 16 16 16 33 33 33 F F F 13 260-279 19 19 19 39 39 39 F F F 14 280-299 12 12 12 25 25 25 F F F 15 300-319 19 19 19 39 39 39 F F F 16 320-339 19 19 19 39 39 39 F F F 17 340-359 17 17 17 36 36 36 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 17 17 17 36 36 36 F F F 22 440-459 19 19 19 39 39 39 F F F 23 460-479 8 8 8 16 16 16 F F F 24 480-499 19 19 19 39 39 39 F F F 25 500-519 19 19 19 39 39 39 F F F 26 520-539 19 19 19 39 39 39 F F F 27 540-559 19 19 19 39 39 39 F F F 28 560-579 18 18 18 38 38 38 F F F 29 580-599 18 18 18 37 37 37 F F F Max gen: 19, Max flush: 39 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.026057215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:16.070217533Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.070584148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.102017861Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.10283128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.102884188Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.102901248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.102923587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} {"msg":"Crucible 95aef0d6-cc68-44ff-be0f-894faeb6ca98 has session id: 945ec363-0722-400b-9773-e31edb394aa2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103462436Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Upstairs opts: Upstairs UUID: 95aef0d6-cc68-44ff-be0f-894faeb6ca98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103528393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Crucible stats registered with UUID: 95aef0d6-cc68-44ff-be0f-894faeb6ca98","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103556452Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:16.103577912Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103764255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103804713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103823712Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.103992566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104015865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} {"msg":"95aef0d6-cc68-44ff-be0f-894faeb6ca98 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104099412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104119881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"0","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.10414565Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"1","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104177729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"2","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104210498Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104242696Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104278085Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47136)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104311124Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40017)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104360902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54587)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.104394251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.105641003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"1","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.105700961Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"2","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.105921172Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"0","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106294248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106325627Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[0]R flush_numbers[0..12]: [39, 29, 37, 36, 28, 1, 36, 35, 36, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106537379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[0]R generation[0..12]: [19, 12, 18, 17, 12, 1, 17, 16, 17, 18, 18, 16]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106570628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106593317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[1]R flush_numbers[0..12]: [39, 29, 37, 36, 28, 1, 36, 35, 36, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106621906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[1]R generation[0..12]: [19, 12, 18, 17, 12, 1, 17, 16, 17, 18, 18, 16]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106665954Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106699073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[2]R flush_numbers[0..12]: [39, 29, 37, 36, 28, 1, 36, 35, 36, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106732782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[2]R generation[0..12]: [19, 12, 18, 17, 12, 1, 17, 16, 17, 18, 18, 16]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.10676366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106784929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:57:16.106811048Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106847787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106874526Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106891525Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"":"downstairs","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106914055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106939963Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"0","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106973312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"1","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.106993581Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"client":"2","":"downstairs","session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.10703691Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.107068569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"95aef0d6-cc68-44ff-be0f-894faeb6ca98 is now active with session: 945ec363-0722-400b-9773-e31edb394aa2","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.107092958Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.107114677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283,"session_id":"945ec363-0722-400b-9773-e31edb394aa2"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.107144076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} {"msg":"Activated sub_volume 95aef0d6-cc68-44ff-be0f-894faeb6ca98","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:16.107166555Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 36864 data: 2 2 2 2 2 2 2 2 3 02/30 Read block 575 len 4096 03/30 Write block 117 len 36864 data: 3 3 3 4 4 4 4 4 4 04/30 Read block 10 len 32768 05/30 Write block 479 len 8192 data: 2 2 06/30 Write block 422 len 16384 data: 6 6 6 3 07/30 Write block 536 len 36864 data: 4 4 4 4 3 4 2 2 2 08/30 Write block 252 len 12288 data: 4 5 5 09/30 Write block 152 len 40960 data: 3 3 2 2 2 2 2 2 2 3 10/30 Write block 314 len 28672 data: 4 4 3 2 2 3 3 11/30 Read block 395 len 36864 12/30 Write block 346 len 12288 data: 4 4 3 13/30 Read block 83 len 20480 14/30 Write block 567 len 28672 data: 3 3 5 4 4 3 3 15/30 Flush 16/30 Write block 221 len 16384 data: 2 3 3 3 17/30 Flush 18/30 Flush 19/30 Write block 294 len 20480 data: 4 4 4 4 4 20/30 Read block 190 len 8192 21/30 Write block 501 len 20480 data: 2 3 3 3 2 22/30 Read block 273 len 12288 23/30 Read block 590 len 36864 24/30 Write block 54 len 32768 data: 4 3 2 2 2 3 4 4 25/30 Write block 77 len 32768 data: 3 3 2 3 3 3 3 3 26/30 Write block 500 len 28672 data: 2 3 4 4 4 3 2 27/30 Read block 201 len 16384 28/30 Write block 427 len 40960 data: 3 3 3 2 2 2 2 2 3 3 29/30 Read block 66 len 4096 30/30 Write block 299 len 16384 data: 4 5 6 5 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:16.127228611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1283} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 39 39 39 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 18 18 18 37 37 37 F T T <--- 3 060-079 17 17 17 36 36 36 F T T <--- 4 080-099 12 12 12 28 28 28 F T T <--- 5 100-119 1 20 20 1 40 40 T F F <--- 6 120-139 17 20 20 36 40 40 T F F <--- 7 140-159 16 20 20 35 40 40 F F F <--- 8 160-179 17 20 20 36 40 40 F F F <--- 9 180-199 18 18 18 38 38 38 F F F 10 200-219 18 18 18 38 38 38 F F F 11 220-239 16 20 20 34 41 41 F F F <--- 12 240-259 16 20 20 33 40 40 F F F <--- 13 260-279 19 19 19 39 39 39 F F F 14 280-299 12 12 12 25 25 25 F T T <--- 15 300-319 19 20 20 39 40 40 F T T <--- 16 320-339 19 20 20 39 40 40 F F F <--- 17 340-359 17 20 20 36 40 40 F F F <--- 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 17 20 20 36 40 40 F T T <--- 22 440-459 19 19 19 39 39 39 F F F 23 460-479 8 20 20 16 40 40 F F F <--- 24 480-499 19 20 20 39 40 40 F F F <--- 25 500-519 19 19 19 39 39 39 F T T <--- 26 520-539 19 20 20 39 40 40 F F F <--- 27 540-559 19 20 20 39 40 40 F F F <--- 28 560-579 18 20 20 38 40 40 F F F <--- 29 580-599 18 18 18 37 37 37 F F F Max gen: 20, Max flush: 41 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 39 39 39 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 21 21 21 42 42 42 F F F 4 080-099 21 21 21 42 42 42 F F F 5 100-119 20 20 20 40 40 40 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 20 20 20 40 40 40 F F F 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 18 18 18 38 38 38 F F F 11 220-239 20 20 20 41 41 41 F F F 12 240-259 20 20 20 40 40 40 F F F 13 260-279 19 19 19 39 39 39 F F F 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 20 20 20 40 40 40 F F F 18 360-379 12 12 12 26 26 26 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 21 21 21 42 42 42 F F F 22 440-459 19 19 19 39 39 39 F F F 23 460-479 20 20 20 40 40 40 F F F 24 480-499 20 20 20 40 40 40 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 20 20 20 40 40 40 F F F 28 560-579 20 20 20 40 40 40 F F F 29 580-599 18 18 18 37 37 37 F F F Max gen: 21, Max flush: 42 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.5939398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:20.636565187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.636919413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.667605694Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.66825559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.668306338Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.668322737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.668345116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} {"msg":"Crucible eeb5b8bb-d6f8-40ab-8f81-6b6524677538 has session id: f56a60ee-27f7-4be5-a20c-908ad92cbc32","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.668947823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Upstairs opts: Upstairs UUID: eeb5b8bb-d6f8-40ab-8f81-6b6524677538, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.668985312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Crucible stats registered with UUID: eeb5b8bb-d6f8-40ab-8f81-6b6524677538","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669004831Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:20.66902082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669243952Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.66928921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669310369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669453234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669473833Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} {"msg":"eeb5b8bb-d6f8-40ab-8f81-6b6524677538 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669541621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.6695617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"0","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669578889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"1","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669595449Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"2","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669611698Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669634437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669671876Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"ds_connection connected from Ok(127.0.0.1:44808)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669703194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"ds_connection connected from Ok(127.0.0.1:56568)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669733733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"ds_connection connected from Ok(127.0.0.1:64090)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.669774932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.670989965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"1","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671023984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"2","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671054653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"0","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671683669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671710478Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[0]R flush_numbers[0..12]: [39, 29, 42, 42, 42, 40, 40, 40, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671734027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[0]R generation[0..12]: [19, 12, 21, 21, 21, 20, 20, 20, 20, 18, 18, 20]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671752787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671772066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[1]R flush_numbers[0..12]: [39, 29, 42, 42, 42, 40, 40, 40, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671790335Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[1]R generation[0..12]: [19, 12, 21, 21, 21, 20, 20, 20, 20, 18, 18, 20]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671808614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671831384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[2]R flush_numbers[0..12]: [39, 29, 42, 42, 42, 40, 40, 40, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671849863Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[2]R generation[0..12]: [19, 12, 21, 21, 21, 20, 20, 20, 20, 18, 18, 20]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671868202Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671887601Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671903641Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.67192029Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.67193628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671959239Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"":"downstairs","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671976018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.671991397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"0","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672008287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"1","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672024996Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"client":"2","":"downstairs","session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672042116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672057985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"eeb5b8bb-d6f8-40ab-8f81-6b6524677538 is now active with session: f56a60ee-27f7-4be5-a20c-908ad92cbc32","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672075644Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672092494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291,"session_id":"f56a60ee-27f7-4be5-a20c-908ad92cbc32"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672108553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} {"msg":"Activated sub_volume eeb5b8bb-d6f8-40ab-8f81-6b6524677538","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:20.672122442Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 200 len 8192 data: 6 4 02/30 Write block 366 len 32768 data: 3 3 3 3 3 3 2 2 03/30 Read block 186 len 28672 04/30 Read block 113 len 16384 05/30 Read block 284 len 32768 06/30 Write block 593 len 16384 data: 2 3 3 3 07/30 Read block 295 len 4096 08/30 Read block 554 len 28672 09/30 Read block 476 len 40960 10/30 Flush 11/30 Write block 251 len 12288 data: 4 5 6 12/30 Write block 6 len 36864 data: 3 3 4 4 3 3 5 5 5 13/30 Read block 220 len 36864 14/30 Read block 577 len 16384 15/30 Read block 567 len 40960 16/30 Read block 591 len 8192 17/30 Read block 131 len 40960 18/30 Read block 74 len 24576 19/30 Read block 271 len 20480 20/30 Read block 205 len 24576 21/30 Read block 442 len 4096 22/30 Write block 336 len 20480 data: 3 4 5 5 5 23/30 Flush 24/30 Write block 434 len 24576 data: 3 4 4 3 4 6 25/30 Read block 379 len 4096 26/30 Read block 319 len 12288 27/30 Write block 100 len 20480 data: 2 2 2 2 2 28/30 Write block 431 len 32768 data: 3 3 3 4 5 5 4 5 29/30 Write block 471 len 12288 data: 2 2 2 30/30 Read block 205 len 4096 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 3 30 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:20.692481167Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1291} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 22 22 39 44 44 F F F <--- 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 21 21 21 42 42 42 F F F 4 080-099 21 21 21 42 42 42 F F F 5 100-119 20 20 20 40 40 40 F T T <--- 6 120-139 20 20 20 40 40 40 F F F 7 140-159 20 20 20 40 40 40 F F F 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 18 22 22 38 43 43 T F F <--- 11 220-239 20 20 20 41 41 41 F F F 12 240-259 20 22 22 40 44 44 F F F <--- 13 260-279 19 19 19 39 39 39 F F F 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 20 22 22 40 44 44 F F F <--- 17 340-359 20 22 22 40 44 44 F F F <--- 18 360-379 12 22 22 26 43 43 T F F <--- 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 21 21 21 42 42 42 F T T <--- 22 440-459 19 19 19 39 39 39 F F F 23 460-479 20 20 20 40 40 40 F T T <--- 24 480-499 20 20 20 40 40 40 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 20 20 20 40 40 40 F F F 28 560-579 20 20 20 40 40 40 F F F 29 580-599 18 22 22 37 43 43 F F F <--- Max gen: 22, Max flush: 44 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 44 44 44 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 21 21 21 42 42 42 F F F 4 080-099 21 21 21 42 42 42 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 20 20 20 40 40 40 F F F 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 22 22 22 43 43 43 F F F 11 220-239 20 20 20 41 41 41 F F F 12 240-259 22 22 22 44 44 44 F F F 13 260-279 19 19 19 39 39 39 F F F 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 22 22 22 44 44 44 F F F 17 340-359 22 22 22 44 44 44 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 23 23 23 45 45 45 F F F 22 440-459 19 19 19 39 39 39 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 20 20 20 40 40 40 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 20 20 20 40 40 40 F F F 28 560-579 20 20 20 40 40 40 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 23, Max flush: 45 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.11603676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.166510857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.167260699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.167309887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.167326097Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.167348266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} {"msg":"Crucible f466e027-be8c-4738-8b1e-5c6eb451f305 has session id: 0a3b8a85-58cf-4fbd-9a4e-faed090c3510","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.167983112Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Upstairs opts: Upstairs UUID: f466e027-be8c-4738-8b1e-5c6eb451f305, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168046139Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Crucible stats registered with UUID: f466e027-be8c-4738-8b1e-5c6eb451f305","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168081778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:25.168109277Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168244742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.16828756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168306659Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168443744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168474283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} {"msg":"f466e027-be8c-4738-8b1e-5c6eb451f305 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.16854797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168568699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"0","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168585809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"1","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168602618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"2","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168619177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168643936Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168675815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"ds_connection connected from Ok(127.0.0.1:56838)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168776161Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"ds_connection connected from Ok(127.0.0.1:63268)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.16882009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"ds_connection connected from Ok(127.0.0.1:51439)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.168853778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.170167528Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"2","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.170202937Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"0","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.17038133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"1","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.170771785Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.170858542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[0]R flush_numbers[0..12]: [44, 29, 42, 42, 42, 45, 40, 40, 40, 38, 43, 41]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171014286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[0]R generation[0..12]: [22, 12, 21, 21, 21, 23, 20, 20, 20, 18, 22, 20]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171038625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171058434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[1]R flush_numbers[0..12]: [44, 29, 42, 42, 42, 45, 40, 40, 40, 38, 43, 41]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171076874Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[1]R generation[0..12]: [22, 12, 21, 21, 21, 23, 20, 20, 20, 18, 22, 20]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171095523Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171126922Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[2]R flush_numbers[0..12]: [44, 29, 42, 42, 42, 45, 40, 40, 40, 38, 43, 41]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.17116152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[2]R generation[0..12]: [22, 12, 21, 21, 21, 23, 20, 20, 20, 18, 22, 20]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171186259Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171219508Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171246447Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171268136Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171300785Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171331824Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"":"downstairs","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171356993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171373542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"0","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171401571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"1","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.17143323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"client":"2","":"downstairs","session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171455769Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171474668Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"f466e027-be8c-4738-8b1e-5c6eb451f305 is now active with session: 0a3b8a85-58cf-4fbd-9a4e-faed090c3510","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171504117Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171532896Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299,"session_id":"0a3b8a85-58cf-4fbd-9a4e-faed090c3510"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171556205Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} {"msg":"Activated sub_volume f466e027-be8c-4738-8b1e-5c6eb451f305","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:25.171573145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 570 len 28672 data: 5 5 4 4 3 4 4 04/30 Read block 589 len 32768 05/30 Read block 182 len 40960 06/30 Read block 474 len 12288 07/30 Read block 214 len 28672 08/30 Write block 488 len 12288 data: 6 6 6 09/30 Read block 518 len 24576 10/30 Read block 455 len 32768 11/30 Read block 275 len 16384 12/30 Write block 148 len 12288 data: 3 3 3 13/30 Read block 587 len 12288 14/30 Read block 204 len 32768 15/30 Write block 444 len 40960 data: 5 4 4 3 3 2 2 2 3 3 16/30 Read block 170 len 28672 17/30 Write block 62 len 24576 data: 4 3 3 2 2 2 18/30 Flush 19/30 Write block 153 len 4096 data: 4 20/30 Read block 329 len 16384 21/30 Flush 22/30 Write block 446 len 32768 data: 5 4 4 3 3 3 4 4 23/30 Read block 37 len 40960 24/30 Write block 236 len 16384 data: 4 4 3 3 25/30 Read block 511 len 20480 26/30 Write block 345 len 24576 data: 4 5 5 4 3 3 27/30 Write block 3 len 36864 data: 2 2 3 4 4 5 5 4 4 28/30 Read block 236 len 40960 29/30 Read block 146 len 16384 30/30 Read block 230 len 36864 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 28 1 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1001)) Some(JobId(1020)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:25.19216058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1299} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 44 44 44 T F T <--- 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 24 21 24 48 42 48 F F F <--- 4 080-099 21 21 21 42 42 42 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 24 20 24 49 40 49 F F F <--- 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 22 22 22 43 43 43 F F F 11 220-239 20 20 20 41 41 41 T F T <--- 12 240-259 22 22 22 44 44 44 F F F 13 260-279 19 19 19 39 39 39 F F F 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 22 22 22 44 44 44 F F F 17 340-359 22 22 22 44 44 44 T F T <--- 18 360-379 22 22 22 43 43 43 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 23 23 23 45 45 45 F F F 22 440-459 24 19 24 48 39 48 T F T <--- 23 460-479 23 23 23 45 45 45 F F F 24 480-499 24 20 24 48 40 48 F F F <--- 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 20 20 20 40 40 40 F F F 28 560-579 24 20 24 48 40 48 F T F <--- 29 580-599 22 22 22 43 43 43 F F F Max gen: 24, Max flush: 49 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 21 21 21 42 42 42 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 22 22 22 43 43 43 F F F 11 220-239 25 25 25 50 50 50 F F F 12 240-259 22 22 22 44 44 44 F F F 13 260-279 19 19 19 39 39 39 F F F 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 22 22 22 44 44 44 F F F 17 340-359 25 25 25 50 50 50 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 13 13 13 30 30 30 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 23 23 23 45 45 45 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 24 24 24 48 48 48 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 20 20 20 40 40 40 F F F 28 560-579 24 24 24 48 48 48 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 25, Max flush: 50 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.577203851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:29.620252421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.620628807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.651401185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.65205168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.652120087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.652140707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.652169885Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} {"msg":"Crucible 1a252879-2222-4240-bb1d-97487e19efe4 has session id: 5448ad46-d876-4769-a67a-b8c106013a4b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.652739884Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Upstairs opts: Upstairs UUID: 1a252879-2222-4240-bb1d-97487e19efe4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.652785002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Crucible stats registered with UUID: 1a252879-2222-4240-bb1d-97487e19efe4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.652810761Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:29.652828551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653079571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653119669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653138219Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653348551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.65337739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} {"msg":"1a252879-2222-4240-bb1d-97487e19efe4 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653453437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653484466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"0","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653511385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"1","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653541913Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"2","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653571142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653604271Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653642529Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61904)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653680788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42462)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653716037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56467)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.653752225Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.654964169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"1","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.654998828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"2","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655083045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"0","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655614454Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.65572853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[0]R flush_numbers[0..12]: [50, 29, 42, 48, 42, 45, 40, 49, 40, 38, 43, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655837946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[0]R generation[0..12]: [25, 12, 21, 24, 21, 23, 20, 24, 20, 18, 22, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655864045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655884264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[1]R flush_numbers[0..12]: [50, 29, 42, 48, 42, 45, 40, 49, 40, 38, 43, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655903323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[1]R generation[0..12]: [25, 12, 21, 24, 21, 23, 20, 24, 20, 18, 22, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655921583Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655944342Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[2]R flush_numbers[0..12]: [50, 29, 42, 48, 42, 45, 40, 49, 40, 38, 43, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.655963151Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[2]R generation[0..12]: [25, 12, 21, 24, 21, 23, 20, 24, 20, 18, 22, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.65598128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.65600073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656016929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656033718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656056737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656072487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"":"downstairs","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656089516Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656104636Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"0","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656121585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"1","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656138354Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"client":"2","":"downstairs","session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656155664Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656171463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"1a252879-2222-4240-bb1d-97487e19efe4 is now active with session: 5448ad46-d876-4769-a67a-b8c106013a4b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656189132Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656206072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307,"session_id":"5448ad46-d876-4769-a67a-b8c106013a4b"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656222281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} {"msg":"Activated sub_volume 1a252879-2222-4240-bb1d-97487e19efe4","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:29.656235991Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 83 len 28672 data: 4 4 3 3 2 2 2 02/30 Read block 458 len 12288 03/30 Read block 241 len 40960 04/30 Read block 119 len 16384 05/30 Write block 340 len 16384 data: 6 4 4 3 06/30 Read block 126 len 24576 07/30 Read block 330 len 40960 08/30 Read block 266 len 8192 09/30 Read block 439 len 40960 10/30 Flush 11/30 Flush 12/30 Write block 496 len 20480 data: 4 4 3 3 3 13/30 Read block 87 len 24576 14/30 Write block 392 len 8192 data: 3 3 15/30 Write block 396 len 40960 data: 3 3 3 3 2 2 3 4 4 6 16/30 Read block 235 len 4096 17/30 Read block 384 len 24576 18/30 Flush 19/30 Write block 503 len 20480 data: 5 5 4 3 2 20/30 Write block 339 len 36864 data: 6 7 5 5 4 4 5 6 6 21/30 Read block 20 len 36864 22/30 Write block 554 len 32768 data: 3 2 2 2 2 3 3 3 23/30 Read block 127 len 32768 24/30 Flush 25/30 Flush 26/30 Write block 276 len 8192 data: 2 2 27/30 Write block 396 len 36864 data: 4 4 4 4 3 3 4 5 5 28/30 Read block 284 len 16384 29/30 Write block 576 len 4096 data: 5 30/30 Read block 512 len 32768 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 10 0 1 11 Done 2 12 11 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:29.676701101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1307} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 26 26 26 51 51 51 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 22 22 22 43 43 43 F F F 11 220-239 25 25 25 50 50 50 F F F 12 240-259 22 22 22 44 44 44 F F F 13 260-279 19 19 19 39 39 39 F T T <--- 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 22 26 26 44 54 54 T F F <--- 17 340-359 26 26 26 51 54 54 T F F <--- 18 360-379 22 22 22 43 43 43 F F F 19 380-399 26 26 26 53 53 53 F T T <--- 20 400-419 26 26 26 53 53 53 F T T <--- 21 420-439 23 23 23 45 45 45 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 26 26 26 53 54 54 T F F <--- 26 520-539 20 20 20 40 40 40 F F F 27 540-559 20 26 26 40 54 54 F F F <--- 28 560-579 24 26 26 48 54 54 F T T <--- 29 580-599 22 22 22 43 43 43 F F F Max gen: 26, Max flush: 54 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 21 21 42 42 42 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 26 26 26 51 51 51 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 20 20 20 40 40 40 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 22 22 22 43 43 43 F F F 11 220-239 25 25 25 50 50 50 F F F 12 240-259 22 22 22 44 44 44 F F F 13 260-279 27 27 27 55 55 55 F F F 14 280-299 21 21 21 42 42 42 F F F 15 300-319 21 21 21 42 42 42 F F F 16 320-339 26 26 26 54 54 54 F F F 17 340-359 26 26 26 54 54 54 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 27 27 27 55 55 55 F F F 20 400-419 27 27 27 55 55 55 F F F 21 420-439 23 23 23 45 45 45 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 26 26 26 54 54 54 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 27, Max flush: 55 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.093574239Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:34.136641768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.137025554Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.167658737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.168320791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.168376439Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.168397788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.168429777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} {"msg":"Crucible f0d126d1-e972-45cb-bfea-f11f1e7a89d0 has session id: 1405f516-86a9-4c2d-931f-1dee2b9b3e39","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.168928618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Upstairs opts: Upstairs UUID: f0d126d1-e972-45cb-bfea-f11f1e7a89d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.168995146Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Crucible stats registered with UUID: f0d126d1-e972-45cb-bfea-f11f1e7a89d0","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169033204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:34.169078542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169304494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169356302Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169385981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.1696758Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169706519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} {"msg":"f0d126d1-e972-45cb-bfea-f11f1e7a89d0 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169777286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169799525Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"0","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169816865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"1","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169833834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"2","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169850383Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169867592Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.169884502Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"ds_connection connected from Ok(127.0.0.1:60858)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.170153732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"ds_connection connected from Ok(127.0.0.1:57588)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.17020299Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"ds_connection connected from Ok(127.0.0.1:49993)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.170248928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.171384605Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"2","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.171418624Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"1","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.171564398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"0","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.171960063Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172078478Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[0]R flush_numbers[0..12]: [50, 29, 42, 48, 51, 45, 40, 49, 40, 38, 43, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172187834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[0]R generation[0..12]: [25, 12, 21, 24, 26, 23, 20, 24, 20, 18, 22, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172216973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172237732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[1]R flush_numbers[0..12]: [50, 29, 42, 48, 51, 45, 40, 49, 40, 38, 43, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172256512Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[1]R generation[0..12]: [25, 12, 21, 24, 26, 23, 20, 24, 20, 18, 22, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172274791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.17229451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[2]R flush_numbers[0..12]: [50, 29, 42, 48, 51, 45, 40, 49, 40, 38, 43, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172312869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[2]R generation[0..12]: [25, 12, 21, 24, 26, 23, 20, 24, 20, 18, 22, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172330939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172350438Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172366607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172383437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172399386Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172415395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"":"downstairs","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172432115Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","Wait for a query_work_queue command to finish before sending IO pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172455864Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"0","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172474593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"1","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172491452Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"client":"2","":"downstairs","session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172508422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172524331Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"f0d126d1-e972-45cb-bfea-f11f1e7a89d0 is now active with session: 1405f516-86a9-4c2d-931f-1dee2b9b3e39","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172542271Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.17255734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315,"session_id":"1405f516-86a9-4c2d-931f-1dee2b9b3e39"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172573329Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} {"msg":"Activated sub_volume f0d126d1-e972-45cb-bfea-f11f1e7a89d0","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:34.172587169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 200 len 28672 data: 7 5 3 3 4 4 3 02/30 Read block 590 len 20480 03/30 Write block 48 len 16384 data: 4 5 4 5 04/30 Read block 584 len 36864 05/30 Read block 290 len 28672 06/30 Read block 135 len 24576 07/30 Read block 215 len 32768 08/30 Read block 524 len 24576 09/30 Read block 377 len 12288 10/30 Read block 389 len 20480 11/30 Read block 292 len 20480 12/30 Read block 194 len 16384 13/30 Write block 429 len 40960 data: 4 3 4 4 4 5 6 6 5 6 14/30 Flush 15/30 Write block 317 len 28672 data: 3 3 4 4 4 4 4 16/30 Write block 255 len 28672 data: 5 5 5 5 4 4 4 17/30 Read block 376 len 16384 18/30 Read block 473 len 28672 19/30 Read block 10 len 32768 20/30 Write block 186 len 8192 data: 4 3 21/30 Write block 282 len 8192 data: 3 3 22/30 Write block 308 len 28672 data: 2 5 5 5 5 4 5 23/30 Write block 162 len 12288 data: 2 2 2 24/30 Read block 488 len 40960 25/30 Write block 92 len 24576 data: 2 2 2 3 3 3 26/30 Read block 578 len 12288 27/30 Read block 545 len 4096 28/30 Read block 299 len 16384 29/30 Read block 70 len 36864 30/30 Write block 282 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:34.193059919Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1315} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 21 28 28 42 56 56 F F F <--- 3 060-079 24 24 24 48 48 48 F F F 4 080-099 26 26 26 51 51 51 F T T <--- 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 20 20 20 40 40 40 F T T <--- 9 180-199 18 18 18 38 38 38 F T T <--- 10 200-219 22 28 28 43 56 56 F F F <--- 11 220-239 25 25 25 50 50 50 F F F 12 240-259 22 22 22 44 44 44 F T T <--- 13 260-279 27 27 27 55 55 55 F T T <--- 14 280-299 21 21 21 42 42 42 F T T <--- 15 300-319 21 21 21 42 42 42 F T T <--- 16 320-339 26 26 26 54 54 54 F T T <--- 17 340-359 26 26 26 54 54 54 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 27 27 27 55 55 55 F F F 20 400-419 27 27 27 55 55 55 F F F 21 420-439 23 28 28 45 56 56 F F F <--- 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 26 26 26 54 54 54 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 28, Max flush: 56 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 28 28 28 56 56 56 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 29 29 29 57 57 57 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 29 29 29 57 57 57 F F F 9 180-199 29 29 29 57 57 57 F F F 10 200-219 28 28 28 56 56 56 F F F 11 220-239 25 25 25 50 50 50 F F F 12 240-259 29 29 29 57 57 57 F F F 13 260-279 29 29 29 57 57 57 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 26 26 26 54 54 54 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 27 27 27 55 55 55 F F F 20 400-419 27 27 27 55 55 55 F F F 21 420-439 28 28 28 56 56 56 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 26 26 26 54 54 54 F F F 26 520-539 20 20 20 40 40 40 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 29, Max flush: 57 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.621760127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.66603858Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.666774012Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.666821721Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.66683779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.666859919Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} {"msg":"Crucible a345f29b-c507-4880-a39d-69f914dee83d has session id: e5df6b19-4240-4372-8657-22518554cde5","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.667457446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Upstairs opts: Upstairs UUID: a345f29b-c507-4880-a39d-69f914dee83d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.667517104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Crucible stats registered with UUID: a345f29b-c507-4880-a39d-69f914dee83d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.667540673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:38.667557073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.667814823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.667857931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.66787676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668083613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668108632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} {"msg":"a345f29b-c507-4880-a39d-69f914dee83d active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668186678Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668207208Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"0","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668224217Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"1","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668241736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"2","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668258056Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668283955Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668314044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44014)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668539295Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35955)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668586673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35864)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.668612652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.669754179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"0","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.669803267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"2","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.670370975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.670550008Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.670974962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"1","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[0]R flush_numbers[0..12]: [50, 29, 56, 48, 57, 45, 40, 49, 57, 57, 56, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671480993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[0]R generation[0..12]: [25, 12, 28, 24, 29, 23, 20, 24, 29, 29, 28, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671506442Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671526701Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[1]R flush_numbers[0..12]: [50, 29, 56, 48, 57, 45, 40, 49, 57, 57, 56, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671545201Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[1]R generation[0..12]: [25, 12, 28, 24, 29, 23, 20, 24, 29, 29, 28, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.6715633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671583129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[2]R flush_numbers[0..12]: [50, 29, 56, 48, 57, 45, 40, 49, 57, 57, 56, 50]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671601368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[2]R generation[0..12]: [25, 12, 28, 24, 29, 23, 20, 24, 29, 29, 28, 25]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671619448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671642437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671659126Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671676036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671698865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671714784Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"":"downstairs","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671732223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671747443Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"0","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671764422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"1","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671781322Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"client":"2","":"downstairs","session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671798481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.67181455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"a345f29b-c507-4880-a39d-69f914dee83d is now active with session: e5df6b19-4240-4372-8657-22518554cde5","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.67183253Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671847469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323,"session_id":"e5df6b19-4240-4372-8657-22518554cde5"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-14T23:57:38.671863349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} {"msg":"Activated sub_volume a345f29b-c507-4880-a39d-69f914dee83d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:38.671881358Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 28672 data: 3 4 4 4 4 4 4 02/30 Flush 03/30 Write block 395 len 36864 data: 3 5 5 5 5 4 4 5 6 04/30 Read block 582 len 32768 05/30 Read block 279 len 16384 06/30 Read block 544 len 12288 07/30 Flush 08/30 Write block 32 len 28672 data: 3 2 2 2 2 2 2 09/30 Read block 327 len 40960 10/30 Read block 378 len 36864 11/30 Read block 136 len 36864 12/30 Flush 13/30 Write block 83 len 24576 data: 5 5 4 4 3 3 14/30 Flush 15/30 Write block 204 len 24576 data: 5 5 4 3 3 3 16/30 Write block 40 len 20480 data: 3 3 3 3 3 17/30 Read block 533 len 8192 18/30 Write block 502 len 20480 data: 5 6 6 5 4 19/30 Write block 230 len 12288 data: 3 3 3 20/30 Read block 137 len 20480 21/30 Read block 183 len 12288 22/30 Read block 388 len 12288 23/30 Write block 121 len 36864 data: 5 5 5 5 5 4 4 4 2 24/30 Write block 10 len 32768 data: 5 5 6 6 6 6 6 5 25/30 Write block 372 len 28672 data: 3 3 2 2 2 2 2 26/30 Read block 541 len 20480 27/30 Write block 269 len 16384 data: 3 2 2 2 28/30 Read block 360 len 12288 29/30 Read block 101 len 32768 30/30 Write block 413 len 12288 data: 3 3 2 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1001)) Some(JobId(1013)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:38.691901055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1323} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 T F T <--- 1 020-039 30 12 30 60 29 60 F F F <--- 2 040-059 28 28 28 56 56 56 T F T <--- 3 060-079 24 24 24 48 48 48 F F F 4 080-099 30 29 30 61 57 61 F F F <--- 5 100-119 23 23 23 45 45 45 F F F 6 120-139 20 20 20 40 40 40 T F T <--- 7 140-159 24 24 24 49 49 49 F F F 8 160-179 29 29 29 57 57 57 F F F 9 180-199 29 29 29 57 57 57 F F F 10 200-219 28 28 28 56 56 56 T F T <--- 11 220-239 25 25 25 50 50 50 T F T <--- 12 240-259 29 29 29 57 57 57 F F F 13 260-279 29 29 29 57 57 57 T F T <--- 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 26 26 26 54 54 54 F F F 18 360-379 22 22 22 43 43 43 T F T <--- 19 380-399 30 27 30 59 55 59 F F F <--- 20 400-419 30 27 30 59 55 59 T F T <--- 21 420-439 28 28 28 56 56 56 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 30 30 30 58 58 58 T F T <--- 26 520-539 30 30 30 58 58 58 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 30, Max flush: 61 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 62 62 62 F F F 1 020-039 30 30 30 60 60 60 F F F 2 040-059 31 31 31 62 62 62 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 31 31 31 62 62 62 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 29 29 29 57 57 57 F F F 9 180-199 29 29 29 57 57 57 F F F 10 200-219 31 31 31 62 62 62 F F F 11 220-239 31 31 31 62 62 62 F F F 12 240-259 29 29 29 57 57 57 F F F 13 260-279 31 31 31 62 62 62 F F F 14 280-299 29 29 29 57 57 57 F F F 15 300-319 29 29 29 57 57 57 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 26 26 26 54 54 54 F F F 18 360-379 31 31 31 62 62 62 F F F 19 380-399 30 30 30 59 59 59 F F F 20 400-419 31 31 31 62 62 62 F F F 21 420-439 28 28 28 56 56 56 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 27 27 27 55 55 55 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 31, Max flush: 62 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.092338399Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.136326133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137050026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137099134Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137115023Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137137003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} {"msg":"Crucible 795c88a7-9ad0-4bea-a098-357f5edba622 has session id: 90bafe29-ce37-401f-a3c0-71498de5bcfe","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137594415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Upstairs opts: Upstairs UUID: 795c88a7-9ad0-4bea-a098-357f5edba622, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137651223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Crucible stats registered with UUID: 795c88a7-9ad0-4bea-a098-357f5edba622","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137683772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:43.137709451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137889464Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137937412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.137969641Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138138935Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138168763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} {"msg":"795c88a7-9ad0-4bea-a098-357f5edba622 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138225761Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.13824773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"0","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138276269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"1","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138294418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"2","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138310938Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138358886Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138395915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:41845)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138429063Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:50729)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138461182Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:36651)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.138491141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.139682686Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"0","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.139732244Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"1","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.14034619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.140374889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.14062126Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"2","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[0]R flush_numbers[0..12]: [62, 60, 62, 48, 61, 45, 62, 49, 57, 57, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.14115892Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[0]R generation[0..12]: [31, 30, 31, 24, 30, 23, 31, 24, 29, 29, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141193058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141218257Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[1]R flush_numbers[0..12]: [62, 60, 62, 48, 61, 45, 62, 49, 57, 57, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141237846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[1]R generation[0..12]: [31, 30, 31, 24, 30, 23, 31, 24, 29, 29, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141256066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141275255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[2]R flush_numbers[0..12]: [62, 60, 62, 48, 61, 45, 62, 49, 57, 57, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141293484Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[2]R generation[0..12]: [31, 30, 31, 24, 30, 23, 31, 24, 29, 29, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141311524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141330823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141347292Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141363902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141380021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"All extents match","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2026-02-14T23:57:43.14140283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"":"downstairs","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141419779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141434819Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"0","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141451188Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"1","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141469388Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"client":"2","":"downstairs","session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141486477Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141502526Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"795c88a7-9ad0-4bea-a098-357f5edba622 is now active with session: 90bafe29-ce37-401f-a3c0-71498de5bcfe","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141520366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141535455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331,"session_id":"90bafe29-ce37-401f-a3c0-71498de5bcfe"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141551394Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} {"msg":"Activated sub_volume 795c88a7-9ad0-4bea-a098-357f5edba622","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:43.141565284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 359 len 12288 data: 3 2 2 02/30 Read block 517 len 16384 03/30 Read block 160 len 12288 04/30 Write block 167 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Flush 06/30 Write block 68 len 32768 data: 2 3 4 4 4 5 4 2 07/30 Read block 74 len 40960 08/30 Flush 09/30 Write block 305 len 36864 data: 3 2 2 3 6 6 6 6 5 10/30 Read block 403 len 40960 11/30 Write block 568 len 16384 data: 4 6 6 6 12/30 Read block 93 len 4096 13/30 Write block 58 len 40960 data: 3 4 5 5 5 4 4 3 3 3 14/30 Read block 296 len 40960 15/30 Flush 16/30 Write block 1 len 32768 data: 3 2 3 3 4 5 5 6 17/30 Write block 223 len 24576 data: 4 4 3 2 2 2 18/30 Read block 228 len 16384 19/30 Write block 88 len 12288 data: 4 3 2 20/30 Read block 167 len 36864 21/30 Read block 24 len 12288 22/30 Write block 445 len 12288 data: 5 6 5 23/30 Write block 384 len 32768 data: 2 2 2 2 2 2 3 3 24/30 Flush 25/30 Write block 293 len 24576 data: 4 5 5 5 5 5 26/30 Write block 77 len 4096 data: 4 27/30 Read block 125 len 16384 28/30 Write block 170 len 20480 data: 3 3 3 3 3 29/30 Read block 259 len 12288 30/30 Write block 424 len 32768 data: 7 4 3 4 4 5 4 5 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 24 24 0 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:43.161702037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1331} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 31 66 66 62 F F F <--- 1 020-039 30 30 30 60 60 60 F F F 2 040-059 32 32 31 65 65 62 F F F <--- 3 060-079 32 32 24 65 65 48 T T F <--- 4 080-099 32 32 30 66 66 61 F F F <--- 5 100-119 23 23 23 45 45 45 F F F 6 120-139 31 31 31 62 62 62 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 32 32 32 63 63 63 T T F <--- 9 180-199 29 29 29 57 57 57 F F F 10 200-219 31 31 31 62 62 62 F F F 11 220-239 32 32 31 66 66 62 F F F <--- 12 240-259 29 29 29 57 57 57 F F F 13 260-279 31 31 31 62 62 62 F F F 14 280-299 29 29 29 57 57 57 T T F <--- 15 300-319 32 32 29 65 65 57 F F F <--- 16 320-339 29 29 29 57 57 57 F F F 17 340-359 32 32 32 63 63 63 F F F 18 360-379 32 32 32 63 63 63 F F F 19 380-399 32 32 30 66 66 59 F F F <--- 20 400-419 31 31 31 62 62 62 F F F 21 420-439 28 28 28 56 56 56 T T F <--- 22 440-459 32 32 25 66 66 50 F F F <--- 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 32 32 27 65 65 55 F F F <--- 29 580-599 22 22 22 43 43 43 F F F Max gen: 32, Max flush: 66 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 66 66 66 F F F 1 020-039 30 30 30 60 60 60 F F F 2 040-059 32 32 32 65 65 65 F F F 3 060-079 33 33 33 67 67 67 F F F 4 080-099 32 32 32 66 66 66 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 31 31 31 62 62 62 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 33 33 33 67 67 67 F F F 9 180-199 29 29 29 57 57 57 F F F 10 200-219 31 31 31 62 62 62 F F F 11 220-239 32 32 32 66 66 66 F F F 12 240-259 29 29 29 57 57 57 F F F 13 260-279 31 31 31 62 62 62 F F F 14 280-299 33 33 33 67 67 67 F F F 15 300-319 32 32 32 65 65 65 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 32 32 32 63 63 63 F F F 18 360-379 32 32 32 63 63 63 F F F 19 380-399 32 32 32 66 66 66 F F F 20 400-419 31 31 31 62 62 62 F F F 21 420-439 33 33 33 67 67 67 F F F 22 440-459 32 32 32 66 66 66 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 26 26 26 54 54 54 F F F 28 560-579 32 32 32 65 65 65 F F F 29 580-599 22 22 22 43 43 43 F F F Max gen: 33, Max flush: 67 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.557400772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.602301601Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603040973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603089841Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603105731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.60312774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} {"msg":"Crucible 50dffa11-8944-45c4-bfa8-67a1b70c236f has session id: 5d9dc4f4-4208-44d4-99be-3333dc867743","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603696378Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Upstairs opts: Upstairs UUID: 50dffa11-8944-45c4-bfa8-67a1b70c236f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603741707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Crucible stats registered with UUID: 50dffa11-8944-45c4-bfa8-67a1b70c236f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603761386Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:47.603780705Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.603998377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604050425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604085593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604277146Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604302335Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} {"msg":"50dffa11-8944-45c4-bfa8-67a1b70c236f active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604365553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604395771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"0","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604416181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"1","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.60443375Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"2","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.60445064Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604467649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604484318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"ds_connection connected from Ok(127.0.0.1:37599)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604502147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"ds_connection connected from Ok(127.0.0.1:38058)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604531406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"ds_connection connected from Ok(127.0.0.1:38261)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.604559315Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.605887585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"0","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.605923783Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"1","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.605945152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"2","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606511801Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606551489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[0]R flush_numbers[0..12]: [66, 60, 65, 67, 66, 45, 62, 49, 67, 57, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606580948Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[0]R generation[0..12]: [32, 30, 32, 33, 32, 23, 31, 24, 33, 29, 31, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606611507Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606642846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[1]R flush_numbers[0..12]: [66, 60, 65, 67, 66, 45, 62, 49, 67, 57, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606673235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[1]R generation[0..12]: [32, 30, 32, 33, 32, 23, 31, 24, 33, 29, 31, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606704934Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606736932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[2]R flush_numbers[0..12]: [66, 60, 65, 67, 66, 45, 62, 49, 67, 57, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606766591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[2]R generation[0..12]: [32, 30, 32, 33, 32, 23, 31, 24, 33, 29, 31, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.60679921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606836598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:57:47.606874917Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606910486Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606941334Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.606973133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"":"downstairs","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607023502Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.60705808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"0","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607092799Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"1","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-14T23:57:47.607126538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"client":"2","":"downstairs","session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607156206Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607173006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"50dffa11-8944-45c4-bfa8-67a1b70c236f is now active with session: 5d9dc4f4-4208-44d4-99be-3333dc867743","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607190995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607207044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607219994Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339,"session_id":"5d9dc4f4-4208-44d4-99be-3333dc867743"} {"msg":"Activated sub_volume 50dffa11-8944-45c4-bfa8-67a1b70c236f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:47.607236153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 28672 data: 5 5 5 5 5 5 4 02/30 Flush 03/30 Flush 04/30 Write block 75 len 16384 data: 3 3 5 4 05/30 Write block 545 len 8192 data: 2 2 06/30 Read block 402 len 16384 07/30 Write block 246 len 16384 data: 3 3 3 3 08/30 Flush 09/30 Write block 394 len 28672 data: 3 4 6 6 6 6 5 10/30 Write block 295 len 36864 data: 6 6 6 6 5 6 7 6 3 11/30 Read block 290 len 32768 12/30 Read block 507 len 12288 13/30 Read block 119 len 8192 14/30 Write block 128 len 20480 data: 5 3 3 3 3 15/30 Flush 16/30 Write block 487 len 8192 data: 5 7 17/30 Read block 208 len 16384 18/30 Write block 195 len 4096 data: 2 19/30 Read block 238 len 36864 20/30 Read block 76 len 40960 21/30 Write block 310 len 24576 data: 7 7 7 6 6 5 22/30 Read block 523 len 40960 23/30 Read block 118 len 40960 24/30 Read block 148 len 28672 25/30 Read block 448 len 16384 26/30 Write block 161 len 32768 data: 4 3 3 3 2 2 3 3 27/30 Write block 592 len 32768 data: 3 3 4 4 4 3 3 3 28/30 Write block 17 len 28672 data: 6 4 4 4 5 5 4 29/30 Read block 356 len 4096 30/30 Write block 174 len 32768 data: 4 3 2 2 2 3 2 2 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:47.626361995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1339} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 66 66 66 T F T <--- 1 020-039 34 30 34 68 60 68 T F T <--- 2 040-059 32 32 32 65 65 65 F F F 3 060-079 34 33 34 70 67 70 F F F <--- 4 080-099 32 32 32 66 66 66 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 34 31 34 71 62 71 F F F <--- 7 140-159 24 24 24 49 49 49 F F F 8 160-179 33 33 33 67 67 67 T F T <--- 9 180-199 29 29 29 57 57 57 T F T <--- 10 200-219 31 31 31 62 62 62 F F F 11 220-239 32 32 32 66 66 66 F F F 12 240-259 34 29 34 70 57 70 F F F <--- 13 260-279 31 31 31 62 62 62 F F F 14 280-299 34 33 34 71 67 71 F F F <--- 15 300-319 34 32 34 71 65 71 T F T <--- 16 320-339 29 29 29 57 57 57 F F F 17 340-359 32 32 32 63 63 63 F F F 18 360-379 32 32 32 63 63 63 F F F 19 380-399 34 32 34 71 66 71 F F F <--- 20 400-419 34 31 34 71 62 71 F F F <--- 21 420-439 33 33 33 67 67 67 F F F 22 440-459 32 32 32 66 66 66 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 26 26 26 53 53 53 T F T <--- 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 34 26 34 70 54 70 F F F <--- 28 560-579 32 32 32 65 65 65 F F F 29 580-599 22 22 22 43 43 43 T F T <--- Max gen: 34, Max flush: 71 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 72 72 72 F F F 1 020-039 35 35 35 72 72 72 F F F 2 040-059 32 32 32 65 65 65 F F F 3 060-079 34 34 34 70 70 70 F F F 4 080-099 32 32 32 66 66 66 F F F 5 100-119 23 23 23 45 45 45 F F F 6 120-139 34 34 34 71 71 71 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 35 35 35 72 72 72 F F F 9 180-199 35 35 35 72 72 72 F F F 10 200-219 31 31 31 62 62 62 F F F 11 220-239 32 32 32 66 66 66 F F F 12 240-259 34 34 34 70 70 70 F F F 13 260-279 31 31 31 62 62 62 F F F 14 280-299 34 34 34 71 71 71 F F F 15 300-319 35 35 35 72 72 72 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 32 32 32 63 63 63 F F F 18 360-379 32 32 32 63 63 63 F F F 19 380-399 34 34 34 71 71 71 F F F 20 400-419 34 34 34 71 71 71 F F F 21 420-439 33 33 33 67 67 67 F F F 22 440-459 32 32 32 66 66 66 F F F 23 460-479 23 23 23 45 45 45 F F F 24 480-499 35 35 35 72 72 72 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 34 34 34 70 70 70 F F F 28 560-579 32 32 32 65 65 65 F F F 29 580-599 35 35 35 72 72 72 F F F Max gen: 35, Max flush: 72 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.039951647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:52.082723417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.083077774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.113938418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.114619272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.11468881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.114716619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.114750647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} {"msg":"Crucible 1f124efa-fee8-4beb-88dd-496d56ee3277 has session id: 70b4fe77-3992-4157-9a1b-97fa07e3b36a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115255368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Upstairs opts: Upstairs UUID: 1f124efa-fee8-4beb-88dd-496d56ee3277, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115301696Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Crucible stats registered with UUID: 1f124efa-fee8-4beb-88dd-496d56ee3277","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115323015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:52.115340805Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115543187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115585736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115606815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115773809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115798228Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} {"msg":"1f124efa-fee8-4beb-88dd-496d56ee3277 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115863075Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115889164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"0","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115909483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"1","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115928412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"2","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115946902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.115970971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.11600353Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51128)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.116060478Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39159)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.116323677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57858)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.116368216Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.11731144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"2","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.117345569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"1","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.117453474Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"0","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.117894758Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.117970835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[0]R flush_numbers[0..12]: [72, 72, 65, 70, 66, 45, 71, 49, 72, 72, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118074101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[0]R generation[0..12]: [35, 35, 32, 34, 32, 23, 34, 24, 35, 35, 31, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.11810053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118121229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[1]R flush_numbers[0..12]: [72, 72, 65, 70, 66, 45, 71, 49, 72, 72, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118139858Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[1]R generation[0..12]: [35, 35, 32, 34, 32, 23, 34, 24, 35, 35, 31, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118158367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118178087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[2]R flush_numbers[0..12]: [72, 72, 65, 70, 66, 45, 71, 49, 72, 72, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118196206Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[2]R generation[0..12]: [35, 35, 32, 34, 32, 23, 34, 24, 35, 35, 31, 32]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118214225Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118236675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:57:52.118265614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118298932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118325581Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.11835561Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"":"downstairs","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118386139Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118412718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"0","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118446487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"1","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118473786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"client":"2","":"downstairs","session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118491645Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118507654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"1f124efa-fee8-4beb-88dd-496d56ee3277 is now active with session: 70b4fe77-3992-4157-9a1b-97fa07e3b36a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118525424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118540503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347,"session_id":"70b4fe77-3992-4157-9a1b-97fa07e3b36a"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118556283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} {"msg":"Activated sub_volume 1f124efa-fee8-4beb-88dd-496d56ee3277","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:52.118570042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 277 len 12288 data: 3 3 3 02/30 Read block 591 len 16384 03/30 Write block 12 len 24576 data: 7 7 7 7 7 7 04/30 Read block 354 len 32768 05/30 Write block 381 len 4096 data: 2 06/30 Write block 231 len 12288 data: 4 4 3 07/30 Read block 130 len 16384 08/30 Read block 454 len 24576 09/30 Read block 7 len 16384 10/30 Read block 525 len 16384 11/30 Write block 282 len 40960 data: 5 5 3 3 2 2 2 2 2 2 12/30 Read block 7 len 24576 13/30 Write block 474 len 16384 data: 2 2 2 2 14/30 Write block 185 len 28672 data: 4 5 4 3 3 2 2 15/30 Read block 36 len 20480 16/30 Flush 17/30 Write block 354 len 40960 data: 3 3 3 3 3 4 3 3 2 2 18/30 Write block 259 len 24576 data: 5 5 5 4 3 3 19/30 Read block 143 len 40960 20/30 Read block 189 len 24576 21/30 Read block 427 len 24576 22/30 Write block 230 len 8192 data: 4 5 23/30 Read block 581 len 24576 24/30 Write block 264 len 40960 data: 4 4 4 3 4 4 3 3 3 2 25/30 Write block 572 len 40960 data: 5 5 4 5 6 4 4 3 3 3 26/30 Write block 59 len 32768 data: 5 6 6 6 5 5 4 4 27/30 Read block 508 len 36864 28/30 Write block 111 len 8192 data: 2 3 29/30 Write block 217 len 32768 data: 2 2 2 2 3 4 5 5 30/30 Write block 53 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 2 31 Done 3 28 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:52.13805386Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1347} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 36 36 72 73 73 T F F <--- 1 020-039 35 35 35 72 72 72 F F F 2 040-059 32 32 32 65 65 65 F T T <--- 3 060-079 34 34 34 70 70 70 F T T <--- 4 080-099 32 32 32 66 66 66 F F F 5 100-119 23 23 23 45 45 45 F T T <--- 6 120-139 34 34 34 71 71 71 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 35 35 35 72 72 72 F F F 9 180-199 35 36 36 72 73 73 F F F <--- 10 200-219 31 31 31 62 62 62 F T T <--- 11 220-239 32 36 36 66 73 73 F T T <--- 12 240-259 34 34 34 70 70 70 F T T <--- 13 260-279 31 36 36 62 73 73 T T T <--- 14 280-299 34 36 36 71 73 73 F F F <--- 15 300-319 35 35 35 72 72 72 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 32 32 32 63 63 63 F T T <--- 18 360-379 32 32 32 63 63 63 F T T <--- 19 380-399 34 36 36 71 73 73 F F F <--- 20 400-419 34 34 34 71 71 71 F F F 21 420-439 33 33 33 67 67 67 F F F 22 440-459 32 32 32 66 66 66 F F F 23 460-479 23 36 36 45 73 73 F F F <--- 24 480-499 35 35 35 72 72 72 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 34 34 34 70 70 70 F F F 28 560-579 32 32 32 65 65 65 F T T <--- 29 580-599 35 35 35 72 72 72 F T T <--- Max gen: 36, Max flush: 73 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 73 73 73 F F F 1 020-039 35 35 35 72 72 72 F F F 2 040-059 37 37 37 74 74 74 F F F 3 060-079 37 37 37 74 74 74 F F F 4 080-099 32 32 32 66 66 66 F F F 5 100-119 37 37 37 74 74 74 F F F 6 120-139 34 34 34 71 71 71 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 35 35 35 72 72 72 F F F 9 180-199 36 36 36 73 73 73 F F F 10 200-219 37 37 37 74 74 74 F F F 11 220-239 37 37 37 74 74 74 F F F 12 240-259 37 37 37 74 74 74 F F F 13 260-279 37 37 37 74 74 74 F F F 14 280-299 36 36 36 73 73 73 F F F 15 300-319 35 35 35 72 72 72 F F F 16 320-339 29 29 29 57 57 57 F F F 17 340-359 37 37 37 74 74 74 F F F 18 360-379 37 37 37 74 74 74 F F F 19 380-399 36 36 36 73 73 73 F F F 20 400-419 34 34 34 71 71 71 F F F 21 420-439 33 33 33 67 67 67 F F F 22 440-459 32 32 32 66 66 66 F F F 23 460-479 36 36 36 73 73 73 F F F 24 480-499 35 35 35 72 72 72 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 34 34 34 70 70 70 F F F 28 560-579 37 37 37 74 74 74 F F F 29 580-599 37 37 37 74 74 74 F F F Max gen: 37, Max flush: 74 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.614488719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.659411088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.660256806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.660303564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.660319503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.660341483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} {"msg":"Crucible e1a6f817-eae7-4a53-af5e-896b99f0aaec has session id: f5d207bb-0b1e-4c4d-9bb5-38a8380327b8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.660922071Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Upstairs opts: Upstairs UUID: e1a6f817-eae7-4a53-af5e-896b99f0aaec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.660983538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Crucible stats registered with UUID: e1a6f817-eae7-4a53-af5e-896b99f0aaec","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661012607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:56.661029696Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.66120536Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661259978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661282717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661426621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.66146276Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} {"msg":"e1a6f817-eae7-4a53-af5e-896b99f0aaec active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661494229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661510768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"0","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661528168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"1","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661544857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"2","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661565966Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661584835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661660992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41119)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661704411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62699)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661776768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55155)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.661822776Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.662949203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"0","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.662998872Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"1","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.663588139Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.663646637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.664365539Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"2","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[0]R flush_numbers[0..12]: [73, 72, 74, 74, 66, 74, 71, 49, 72, 73, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.66487058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[0]R generation[0..12]: [36, 35, 37, 37, 32, 37, 34, 24, 35, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.664900099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.664921038Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[1]R flush_numbers[0..12]: [73, 72, 74, 74, 66, 74, 71, 49, 72, 73, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.664940078Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[1]R generation[0..12]: [36, 35, 37, 37, 32, 37, 34, 24, 35, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.664958537Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.664978346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[2]R flush_numbers[0..12]: [73, 72, 74, 74, 66, 74, 71, 49, 72, 73, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665010745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[2]R generation[0..12]: [36, 35, 37, 37, 32, 37, 34, 24, 35, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665031804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2026-02-14T23:57:56.665054933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665078972Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665095661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665111631Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.66512755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"":"downstairs","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.66514451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665159509Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"0","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665176178Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"1","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665192588Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"client":"2","":"downstairs","session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665209397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665225257Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"e1a6f817-eae7-4a53-af5e-896b99f0aaec is now active with session: f5d207bb-0b1e-4c4d-9bb5-38a8380327b8","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665242996Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665261525Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355,"session_id":"f5d207bb-0b1e-4c4d-9bb5-38a8380327b8"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665277575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} {"msg":"Activated sub_volume e1a6f817-eae7-4a53-af5e-896b99f0aaec","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:57:56.665291324Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 53 len 28672 data: 5 5 4 3 3 4 6 04/30 Read block 503 len 32768 05/30 Write block 121 len 32768 data: 6 6 6 6 6 5 5 6 06/30 Write block 298 len 32768 data: 7 6 7 8 7 4 3 4 07/30 Write block 347 len 32768 data: 7 5 4 4 3 4 4 4 08/30 Read block 12 len 28672 09/30 Write block 52 len 12288 data: 5 6 6 10/30 Read block 58 len 4096 11/30 Write block 37 len 16384 data: 3 3 2 4 12/30 Read block 236 len 16384 13/30 Write block 549 len 20480 data: 3 3 3 3 3 14/30 Write block 307 len 8192 data: 3 4 15/30 Write block 337 len 8192 data: 5 6 16/30 Read block 312 len 28672 17/30 Read block 29 len 16384 18/30 Read block 214 len 20480 19/30 Read block 301 len 8192 20/30 Write block 403 len 36864 data: 7 6 7 5 5 5 3 3 3 21/30 Write block 409 len 24576 data: 4 4 4 3 4 4 22/30 Read block 82 len 4096 23/30 Write block 280 len 16384 data: 3 3 6 6 24/30 Write block 269 len 36864 data: 5 4 4 4 3 2 2 3 4 25/30 Read block 55 len 36864 26/30 Write block 418 len 40960 data: 4 4 4 5 7 7 8 5 4 5 27/30 Write block 19 len 12288 data: 5 5 6 28/30 Write block 170 len 28672 data: 4 4 4 4 5 4 3 29/30 Write block 580 len 32768 data: 4 4 3 4 4 3 3 4 30/30 Write block 195 len 12288 data: 3 3 4 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 7 Done Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 29 34 Done 27 26 0 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:57:56.682813267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1355} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 73 73 73 T T F <--- 1 020-039 35 35 35 72 72 72 T T F <--- 2 040-059 37 37 37 74 74 74 T T F <--- 3 060-079 37 37 37 74 74 74 F F F 4 080-099 32 32 32 66 66 66 F F F 5 100-119 37 37 37 74 74 74 F F F 6 120-139 34 34 34 71 71 71 T T F <--- 7 140-159 24 24 24 49 49 49 F F F 8 160-179 35 35 35 72 72 72 T T F <--- 9 180-199 36 36 36 73 73 73 T T F <--- 10 200-219 37 37 37 74 74 74 F F F 11 220-239 37 37 37 74 74 74 F F F 12 240-259 37 37 37 74 74 74 F F F 13 260-279 37 37 37 74 74 74 T T F <--- 14 280-299 36 36 36 73 73 73 T T F <--- 15 300-319 35 35 35 72 72 72 T T F <--- 16 320-339 29 29 29 57 57 57 T T F <--- 17 340-359 37 37 37 74 74 74 T T F <--- 18 360-379 37 37 37 74 74 74 F F F 19 380-399 36 36 36 73 73 73 F F F 20 400-419 34 34 34 71 71 71 T T F <--- 21 420-439 33 33 33 67 67 67 T T F <--- 22 440-459 32 32 32 66 66 66 F F F 23 460-479 36 36 36 73 73 73 F F F 24 480-499 35 35 35 72 72 72 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 34 34 34 70 70 70 T T F <--- 28 560-579 37 37 37 74 74 74 F F F 29 580-599 37 37 37 74 74 74 T T F <--- Max gen: 37, Max flush: 74 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 75 75 75 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 37 37 37 74 74 74 F F F 4 080-099 32 32 32 66 66 66 F F F 5 100-119 37 37 37 74 74 74 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 24 24 24 49 49 49 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 37 37 37 74 74 74 F F F 11 220-239 37 37 37 74 74 74 F F F 12 240-259 37 37 37 74 74 74 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 75 75 75 F F F 17 340-359 38 38 38 75 75 75 F F F 18 360-379 37 37 37 74 74 74 F F F 19 380-399 36 36 36 73 73 73 F F F 20 400-419 38 38 38 75 75 75 F F F 21 420-439 38 38 38 75 75 75 F F F 22 440-459 32 32 32 66 66 66 F F F 23 460-479 36 36 36 73 73 73 F F F 24 480-499 35 35 35 72 72 72 F F F 25 500-519 31 31 31 62 62 62 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 38 38 38 75 75 75 F F F 28 560-579 37 37 37 74 74 74 F F F 29 580-599 38 38 38 75 75 75 F F F Max gen: 38, Max flush: 75 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.104653375Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.149866873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.150577266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.150624804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.150640944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.150662963Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} {"msg":"Crucible e28a0b80-fddf-4f14-8197-7d3743168602 has session id: 80783fa5-8803-4a6b-9a10-8a6be8a8dd41","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151220721Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Upstairs opts: Upstairs UUID: e28a0b80-fddf-4f14-8197-7d3743168602, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151282809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Crucible stats registered with UUID: e28a0b80-fddf-4f14-8197-7d3743168602","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151313998Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:01.151341187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151621676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151671384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151702883Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151830558Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151860157Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} {"msg":"e28a0b80-fddf-4f14-8197-7d3743168602 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151922195Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151944744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"0","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151967943Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"1","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.151998762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"2","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.15203399Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.152073739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.152108188Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"ds_connection connected from Ok(127.0.0.1:37261)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.152274501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"ds_connection connected from Ok(127.0.0.1:40592)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.152324639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"ds_connection connected from Ok(127.0.0.1:39346)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.152358178Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.153443347Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"0","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.153534033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"1","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.153564962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"2","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154013455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154112501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[0]R flush_numbers[0..12]: [75, 75, 75, 74, 66, 74, 75, 49, 75, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154242136Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[0]R generation[0..12]: [38, 38, 38, 37, 32, 37, 38, 24, 38, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154270905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154291744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[1]R flush_numbers[0..12]: [75, 75, 75, 74, 66, 74, 75, 49, 75, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154310184Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[1]R generation[0..12]: [38, 38, 38, 37, 32, 37, 38, 24, 38, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154328263Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154347632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[2]R flush_numbers[0..12]: [75, 75, 75, 74, 66, 74, 75, 49, 75, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154365982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[2]R generation[0..12]: [38, 38, 38, 37, 32, 37, 38, 24, 38, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154384141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.15440346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Max found gen is 39","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154419829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Generation requested: 40 >= found:39","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154436709Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154459488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154475587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"":"downstairs","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154492577Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154507676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"0","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154526425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"1","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154543455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"client":"2","":"downstairs","session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154560244Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154576283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"e28a0b80-fddf-4f14-8197-7d3743168602 is now active with session: 80783fa5-8803-4a6b-9a10-8a6be8a8dd41","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154594133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154613222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363,"session_id":"80783fa5-8803-4a6b-9a10-8a6be8a8dd41"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154629081Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} {"msg":"Activated sub_volume e28a0b80-fddf-4f14-8197-7d3743168602","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:01.154642821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 28672 data: 5 5 6 5 4 3 3 02/30 Write block 560 len 16384 data: 4 4 3 4 03/30 Read block 355 len 12288 04/30 Write block 135 len 36864 data: 4 4 4 4 3 3 3 3 2 05/30 Write block 374 len 24576 data: 3 3 3 3 3 2 06/30 Read block 352 len 36864 07/30 Read block 522 len 20480 08/30 Write block 89 len 28672 data: 4 3 2 3 3 3 4 09/30 Read block 546 len 8192 10/30 Flush 11/30 Write block 440 len 8192 data: 6 5 12/30 Read block 105 len 28672 13/30 Read block 207 len 20480 14/30 Write block 505 len 28672 data: 6 5 3 2 2 2 2 15/30 Read block 78 len 36864 16/30 Read block 517 len 36864 17/30 Write block 431 len 4096 data: 6 18/30 Write block 456 len 32768 data: 3 2 2 2 2 2 2 2 19/30 Flush 20/30 Write block 252 len 32768 data: 6 7 6 6 6 6 6 6 21/30 Read block 175 len 16384 22/30 Read block 238 len 24576 23/30 Write block 487 len 16384 data: 6 8 7 7 24/30 Write block 442 len 32768 data: 4 4 6 6 7 6 5 4 25/30 Flush 26/30 Write block 322 len 20480 data: 5 5 2 2 2 27/30 Write block 60 len 40960 data: 7 7 7 6 6 5 5 4 3 4 28/30 Read block 216 len 28672 29/30 Read block 542 len 32768 30/30 Write block 313 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:01.175022065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1363} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 75 75 75 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 37 37 37 74 74 74 T F T <--- 4 080-099 40 32 40 76 66 76 F F F <--- 5 100-119 37 37 37 74 74 74 F F F 6 120-139 40 38 40 76 75 76 F F F <--- 7 140-159 40 24 40 76 49 76 F F F <--- 8 160-179 40 38 40 76 75 76 F F F <--- 9 180-199 38 38 38 75 75 75 F F F 10 200-219 37 37 37 74 74 74 F F F 11 220-239 37 37 37 74 74 74 F F F 12 240-259 40 37 40 78 74 78 F F F <--- 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 38 38 38 75 75 75 T F T <--- 16 320-339 38 38 38 75 75 75 T F T <--- 17 340-359 38 38 38 75 75 75 F F F 18 360-379 40 37 40 76 74 76 F F F <--- 19 380-399 36 36 36 73 73 73 F F F 20 400-419 38 38 38 75 75 75 F F F 21 420-439 40 38 40 77 75 77 F F F <--- 22 440-459 40 32 40 78 66 78 F F F <--- 23 460-479 40 36 40 77 73 77 F F F <--- 24 480-499 40 35 40 78 72 78 F F F <--- 25 500-519 40 31 40 77 62 77 F F F <--- 26 520-539 30 30 30 58 58 58 F F F 27 540-559 38 38 38 75 75 75 F F F 28 560-579 40 37 40 76 74 76 F F F <--- 29 580-599 38 38 38 75 75 75 F F F Max gen: 40, Max flush: 78 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 75 75 75 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 41 41 41 79 79 79 F F F 4 080-099 40 40 40 76 76 76 F F F 5 100-119 37 37 37 74 74 74 F F F 6 120-139 40 40 40 76 76 76 F F F 7 140-159 40 40 40 76 76 76 F F F 8 160-179 40 40 40 76 76 76 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 37 37 37 74 74 74 F F F 11 220-239 37 37 37 74 74 74 F F F 12 240-259 40 40 40 78 78 78 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 41 41 41 79 79 79 F F F 16 320-339 41 41 41 79 79 79 F F F 17 340-359 38 38 38 75 75 75 F F F 18 360-379 40 40 40 76 76 76 F F F 19 380-399 36 36 36 73 73 73 F F F 20 400-419 38 38 38 75 75 75 F F F 21 420-439 40 40 40 77 77 77 F F F 22 440-459 40 40 40 78 78 78 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 40 40 40 78 78 78 F F F 25 500-519 40 40 40 77 77 77 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 38 38 38 75 75 75 F F F 28 560-579 40 40 40 76 76 76 F F F 29 580-599 38 38 38 75 75 75 F F F Max gen: 41, Max flush: 79 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.583142856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.628769628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.629607976Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.629670254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.629690533Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.629717982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} {"msg":"Crucible e08a0e4d-7007-41b4-bcbc-1d0b88bfa944 has session id: 6d82752b-9250-4bb0-b115-e2216ee9b67f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630163835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Upstairs opts: Upstairs UUID: e08a0e4d-7007-41b4-bcbc-1d0b88bfa944, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630266421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Crucible stats registered with UUID: e08a0e4d-7007-41b4-bcbc-1d0b88bfa944","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630325119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:05.630374027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630570659Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630623457Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630655736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630850729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630889497Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} {"msg":"e08a0e4d-7007-41b4-bcbc-1d0b88bfa944 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630919506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630936585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"0","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630953865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"1","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630970514Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"2","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.630986934Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.631004083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.631021142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54813)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.631128458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40403)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.631176166Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40080)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.631214575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.63265271Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"2","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.632691809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"0","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.632712798Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"1","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633243348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633335614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[0]R flush_numbers[0..12]: [75, 75, 75, 79, 76, 74, 76, 76, 76, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633425581Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[0]R generation[0..12]: [38, 38, 38, 41, 40, 37, 40, 40, 40, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.63344993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633469989Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[1]R flush_numbers[0..12]: [75, 75, 75, 79, 76, 74, 76, 76, 76, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633488218Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[1]R generation[0..12]: [38, 38, 38, 41, 40, 37, 40, 40, 40, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633506367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633525997Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[2]R flush_numbers[0..12]: [75, 75, 75, 79, 76, 74, 76, 76, 76, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633544216Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[2]R generation[0..12]: [38, 38, 38, 41, 40, 37, 40, 40, 40, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633567245Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633587104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633603254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633620013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633636063Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633652242Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"":"downstairs","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633669241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633684481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"0","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.63370149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"1","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Transition from WaitQuorum to Active","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-02-14T23:58:05.633725519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"client":"2","":"downstairs","session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633742709Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633758468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"e08a0e4d-7007-41b4-bcbc-1d0b88bfa944 is now active with session: 6d82752b-9250-4bb0-b115-e2216ee9b67f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633776397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633791467Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371,"session_id":"6d82752b-9250-4bb0-b115-e2216ee9b67f"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633807346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} {"msg":"Activated sub_volume e08a0e4d-7007-41b4-bcbc-1d0b88bfa944","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:05.633820926Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 12288 data: 3 5 4 02/30 Write block 2 len 4096 data: 3 03/30 Write block 434 len 16384 data: 6 7 7 6 04/30 Read block 567 len 16384 05/30 Write block 500 len 28672 data: 4 4 6 7 7 7 6 06/30 Read block 394 len 16384 07/30 Write block 356 len 40960 data: 4 4 4 5 4 4 3 3 3 3 08/30 Write block 72 len 32768 data: 5 6 5 4 4 6 5 3 09/30 Write block 9 len 36864 data: 6 6 6 8 8 8 8 8 8 10/30 Read block 192 len 20480 11/30 Write block 557 len 28672 data: 3 3 4 5 5 4 5 12/30 Write block 227 len 20480 data: 3 3 3 5 6 13/30 Write block 66 len 36864 data: 6 5 4 5 5 5 6 7 6 14/30 Flush 15/30 Write block 557 len 32768 data: 4 4 5 6 6 5 6 4 16/30 Write block 542 len 8192 data: 3 3 17/30 Write block 541 len 16384 data: 5 4 4 3 18/30 Read block 532 len 4096 19/30 Flush 20/30 Write block 77 len 16384 data: 7 6 4 4 21/30 Write block 159 len 40960 data: 3 4 6 5 4 4 3 3 4 4 22/30 Read block 138 len 28672 23/30 Read block 51 len 8192 24/30 Read block 251 len 36864 25/30 Read block 46 len 8192 26/30 Read block 315 len 12288 27/30 Write block 597 len 4096 data: 4 28/30 Write block 588 len 24576 data: 4 4 5 3 4 4 29/30 Read block 72 len 20480 30/30 Write block 335 len 8192 data: 3 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:05.652324911Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1371} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 38 42 80 75 80 F F F <--- 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 42 41 42 80 79 80 T F T <--- 4 080-099 40 40 40 76 76 76 T F T <--- 5 100-119 37 37 37 74 74 74 F F F 6 120-139 40 40 40 76 76 76 F F F 7 140-159 40 40 40 76 76 76 T F T <--- 8 160-179 42 40 42 80 76 80 T T T <--- 9 180-199 38 38 38 75 75 75 F F F 10 200-219 37 37 37 74 74 74 F F F 11 220-239 42 37 42 80 74 80 F F F <--- 12 240-259 40 40 40 78 78 78 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 41 41 41 79 79 79 F F F 16 320-339 41 41 41 79 79 79 T F T <--- 17 340-359 42 38 42 80 75 80 F F F <--- 18 360-379 42 40 42 80 76 80 F F F <--- 19 380-399 36 36 36 73 73 73 F F F 20 400-419 38 38 38 75 75 75 F F F 21 420-439 42 40 42 80 77 80 F F F <--- 22 440-459 40 40 40 78 78 78 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 40 40 40 78 78 78 F F F 25 500-519 42 40 42 80 77 80 F F F <--- 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 38 42 81 75 81 F F F <--- 28 560-579 42 40 42 81 76 81 F F F <--- 29 580-599 38 38 38 75 75 75 T F T <--- Max gen: 42, Max flush: 81 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 43 43 43 82 82 82 F F F 5 100-119 37 37 37 74 74 74 F F F 6 120-139 40 40 40 76 76 76 F F F 7 140-159 43 43 43 82 82 82 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 37 37 37 74 74 74 F F F 11 220-239 42 42 42 80 80 80 F F F 12 240-259 40 40 40 78 78 78 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 41 41 41 79 79 79 F F F 16 320-339 43 43 43 82 82 82 F F F 17 340-359 42 42 42 80 80 80 F F F 18 360-379 42 42 42 80 80 80 F F F 19 380-399 36 36 36 73 73 73 F F F 20 400-419 38 38 38 75 75 75 F F F 21 420-439 42 42 42 80 80 80 F F F 22 440-459 40 40 40 78 78 78 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 40 40 40 78 78 78 F F F 25 500-519 42 42 42 80 80 80 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 42 42 81 81 81 F F F 28 560-579 42 42 42 81 81 81 F F F 29 580-599 43 43 43 82 82 82 F F F Max gen: 43, Max flush: 82 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.06601614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:10.109265782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.109678007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.141502734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.142249836Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.142300164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.142317333Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.142344652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} {"msg":"Crucible 7e3e99ec-af0a-4f7f-ba08-0d32cb5d90cb has session id: ef451ab1-696b-44f5-b5ae-3885397faa6b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.142871392Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Upstairs opts: Upstairs UUID: 7e3e99ec-af0a-4f7f-ba08-0d32cb5d90cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.14294282Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Crucible stats registered with UUID: 7e3e99ec-af0a-4f7f-ba08-0d32cb5d90cb","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.142971258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:10.142988548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.14319292Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143237748Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143260377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143432591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143471269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} {"msg":"7e3e99ec-af0a-4f7f-ba08-0d32cb5d90cb active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143540077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143562536Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"0","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143580415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"1","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143597384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"2","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143614234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143631903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143648822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63223)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143805367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57677)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143856015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40084)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.143898313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.145185424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"1","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.145227702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"2","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.145530681Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"0","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.145797621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.145898987Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[0]R flush_numbers[0..12]: [80, 75, 75, 82, 82, 74, 76, 82, 82, 75, 74, 80]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146231194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[0]R generation[0..12]: [42, 38, 38, 43, 43, 37, 40, 43, 43, 38, 37, 42]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146259843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146284032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[1]R flush_numbers[0..12]: [80, 75, 75, 82, 82, 74, 76, 82, 82, 75, 74, 80]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146307101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[1]R generation[0..12]: [42, 38, 38, 43, 43, 37, 40, 43, 43, 38, 37, 42]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.14632976Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146354419Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[2]R flush_numbers[0..12]: [80, 75, 75, 82, 82, 74, 76, 82, 82, 75, 74, 80]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146377399Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[2]R generation[0..12]: [42, 38, 38, 43, 43, 37, 40, 43, 43, 38, 37, 42]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146400358Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146424637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:58:10.146445046Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146471365Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146491454Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146511904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"":"downstairs","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146536483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146555752Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"0","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146576471Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"1","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.14659702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"client":"2","":"downstairs","session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146618249Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146638299Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 7e3e99ec-af0a-4f7f-ba08-0d32cb5d90cb is now active with session: ef451ab1-696b-44f5-b5ae-3885397faa6b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146663878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146683047Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379,"session_id":"ef451ab1-696b-44f5-b5ae-3885397faa6b"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146703076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} {"msg":"Activated sub_volume 7e3e99ec-af0a-4f7f-ba08-0d32cb5d90cb","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:10.146720616Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 482 len 28672 data: 3 4 5 6 5 7 9 02/30 Read block 475 len 24576 03/30 Read block 236 len 36864 04/30 Write block 126 len 32768 data: 6 6 7 4 4 4 4 3 05/30 Write block 197 len 4096 data: 5 06/30 Read block 449 len 32768 07/30 Read block 60 len 40960 08/30 Write block 477 len 32768 data: 3 2 3 3 3 4 5 6 09/30 Read block 32 len 20480 10/30 Write block 390 len 28672 data: 4 4 4 4 4 5 7 11/30 Write block 312 len 40960 data: 8 8 7 6 4 4 4 5 5 5 12/30 Read block 163 len 12288 13/30 Write block 215 len 4096 data: 2 14/30 Write block 415 len 12288 data: 3 2 4 15/30 Read block 336 len 36864 16/30 Write block 112 len 36864 data: 4 3 3 3 3 4 4 4 5 17/30 Read block 349 len 40960 18/30 Write block 425 len 32768 data: 6 5 6 5 6 5 7 5 19/30 Read block 129 len 40960 20/30 Write block 134 len 36864 data: 3 5 5 5 5 4 4 4 4 21/30 Read block 592 len 24576 22/30 Read block 12 len 16384 23/30 Read block 234 len 4096 24/30 Write block 205 len 20480 data: 6 5 4 4 4 25/30 Read block 396 len 12288 26/30 Write block 136 len 16384 data: 6 6 6 5 27/30 Read block 543 len 20480 28/30 Read block 174 len 24576 29/30 Write block 237 len 40960 data: 5 4 4 2 2 2 2 3 3 4 30/30 Write block 344 len 40960 data: 5 6 7 8 6 5 5 4 5 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:10.168189448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1379} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 43 43 43 82 82 82 F F F 5 100-119 37 37 37 74 74 74 F T T <--- 6 120-139 40 40 40 76 76 76 F T T <--- 7 140-159 43 43 43 82 82 82 F T T <--- 8 160-179 43 43 43 82 82 82 F F F 9 180-199 38 38 38 75 75 75 F T T <--- 10 200-219 37 37 37 74 74 74 F T T <--- 11 220-239 42 42 42 80 80 80 F T T <--- 12 240-259 40 40 40 78 78 78 F T T <--- 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 41 41 41 79 79 79 F T T <--- 16 320-339 43 43 43 82 82 82 F T T <--- 17 340-359 42 42 42 80 80 80 F T T <--- 18 360-379 42 42 42 80 80 80 F F F 19 380-399 36 36 36 73 73 73 F T T <--- 20 400-419 38 38 38 75 75 75 F T T <--- 21 420-439 42 42 42 80 80 80 F T T <--- 22 440-459 40 40 40 78 78 78 F F F 23 460-479 40 40 40 77 77 77 F T T <--- 24 480-499 40 40 40 78 78 78 T T T 25 500-519 42 42 42 80 80 80 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 42 42 81 81 81 F F F 28 560-579 42 42 42 81 81 81 F F F 29 580-599 43 43 43 82 82 82 F F F Max gen: 43, Max flush: 82 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 22 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 483,484d482 < 2, < 2, 485a484 > 3, 488a488 > 6, diff found after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 43 43 43 82 82 82 F F F 5 100-119 44 44 44 83 83 83 F F F 6 120-139 44 44 44 83 83 83 F F F 7 140-159 44 44 44 83 83 83 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 44 44 44 83 83 83 F F F 10 200-219 44 44 44 83 83 83 F F F 11 220-239 44 44 44 83 83 83 F F F 12 240-259 44 44 44 83 83 83 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 44 44 44 83 83 83 F F F 16 320-339 44 44 44 83 83 83 F F F 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 80 80 80 F F F 19 380-399 44 44 44 83 83 83 F F F 20 400-419 44 44 44 83 83 83 F F F 21 420-439 44 44 44 83 83 83 F F F 22 440-459 40 40 40 78 78 78 F F F 23 460-479 44 44 44 83 83 83 F F F 24 480-499 44 44 44 83 83 83 F F F 25 500-519 42 42 42 80 80 80 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 42 42 81 81 81 F F F 28 560-579 42 42 42 81 81 81 F F F 29 580-599 43 43 43 82 82 82 F F F Max gen: 44, Max flush: 83 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.667863901Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.712554889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.713276372Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.71332343Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.713346299Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.713372518Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} {"msg":"Crucible ee3534d8-4e35-41e8-93c7-16331fa97d0b has session id: 2a5576c1-28bb-44a5-a319-82f4e3b0e70b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.713933427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Upstairs opts: Upstairs UUID: ee3534d8-4e35-41e8-93c7-16331fa97d0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.713988364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Crucible stats registered with UUID: ee3534d8-4e35-41e8-93c7-16331fa97d0b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714019633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:14.714045402Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714248145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714302933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714324792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714510465Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714534234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} {"msg":"ee3534d8-4e35-41e8-93c7-16331fa97d0b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714608471Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.71463768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"0","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714669429Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"1","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714701237Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"2","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714730246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714771615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.714801993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40435)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.71488844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52124)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.715027455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62099)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.715071463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.716224499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"1","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.716265028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"0","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.716561737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"2","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.716805907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.716849096Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[0]R flush_numbers[0..12]: [80, 75, 75, 82, 82, 83, 83, 83, 82, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717174153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[0]R generation[0..12]: [42, 38, 38, 43, 43, 44, 44, 44, 43, 44, 44, 44]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717212202Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717236821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[1]R flush_numbers[0..12]: [80, 75, 75, 82, 82, 83, 83, 83, 82, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.7172597Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[1]R generation[0..12]: [42, 38, 38, 43, 43, 44, 44, 44, 43, 44, 44, 44]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717281649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717311678Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[2]R flush_numbers[0..12]: [80, 75, 75, 82, 82, 83, 83, 83, 82, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717334397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[2]R generation[0..12]: [42, 38, 38, 43, 43, 44, 44, 44, 43, 44, 44, 44]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717356196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717375895Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Max found gen is 45","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-02-14T23:58:14.717399675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Generation requested: 46 >= found:45","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717416774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717432633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717448513Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"":"downstairs","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717465362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717480442Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"0","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717497481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"1","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.71751425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"client":"2","":"downstairs","session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.71753133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717557789Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"ee3534d8-4e35-41e8-93c7-16331fa97d0b is now active with session: 2a5576c1-28bb-44a5-a319-82f4e3b0e70b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717576278Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717593387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390,"session_id":"2a5576c1-28bb-44a5-a319-82f4e3b0e70b"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717609517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} {"msg":"Activated sub_volume ee3534d8-4e35-41e8-93c7-16331fa97d0b","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:14.717623266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 298 len 4096 data: 8 02/30 Flush 03/30 Write block 395 len 12288 data: 6 8 7 04/30 Read block 570 len 36864 05/30 Write block 188 len 16384 data: 4 4 3 3 06/30 Read block 118 len 40960 07/30 Read block 453 len 28672 08/30 Write block 203 len 28672 data: 4 6 7 6 5 5 5 09/30 Write block 216 len 36864 data: 2 3 3 3 3 4 5 6 6 10/30 Read block 62 len 36864 11/30 Read block 431 len 20480 12/30 Read block 93 len 36864 13/30 Read block 71 len 16384 14/30 Write block 116 len 20480 data: 4 5 5 5 6 15/30 Write block 116 len 40960 data: 5 6 6 6 7 7 7 7 7 7 16/30 Write block 55 len 24576 data: 5 4 4 5 7 8 17/30 Read block 460 len 8192 18/30 Write block 337 len 8192 data: 6 7 19/30 Read block 428 len 40960 20/30 Flush 21/30 Write block 202 len 4096 data: 4 22/30 Write block 194 len 24576 data: 2 4 4 6 5 5 23/30 Read block 328 len 12288 24/30 Read block 100 len 4096 25/30 Write block 518 len 4096 data: 5 26/30 Read block 351 len 20480 27/30 Write block 502 len 40960 data: 7 8 8 8 7 4 3 3 3 3 28/30 Write block 230 len 12288 data: 6 7 5 29/30 Write block 148 len 36864 data: 4 4 4 3 4 5 3 3 3 30/30 Write block 420 len 8192 data: 5 6 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 10 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 27 35 Done 24 24 1 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:14.735446267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1390} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 46 46 38 85 85 75 F F F <--- 3 060-079 46 46 43 85 85 82 F F F <--- 4 080-099 43 43 43 82 82 82 F F F 5 100-119 46 46 44 85 85 83 F F F <--- 6 120-139 46 46 44 85 85 83 F F F <--- 7 140-159 44 44 44 83 83 83 T T F <--- 8 160-179 43 43 43 82 82 82 F F F 9 180-199 46 46 44 85 85 83 T T F <--- 10 200-219 46 46 44 85 85 83 T T F <--- 11 220-239 46 46 44 85 85 83 T T F <--- 12 240-259 44 44 44 83 83 83 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 46 46 46 84 84 84 F F F 15 300-319 44 44 44 83 83 83 F F F 16 320-339 46 46 44 85 85 83 F F F <--- 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 80 80 80 F F F 19 380-399 46 46 44 85 85 83 F F T <--- 20 400-419 44 44 44 83 83 83 F F F 21 420-439 44 44 44 83 83 83 T T F <--- 22 440-459 40 40 40 78 78 78 F F F 23 460-479 44 44 44 83 83 83 F F F 24 480-499 44 44 44 83 83 83 F F F 25 500-519 42 42 42 80 80 80 T T F <--- 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 42 42 81 81 81 F F F 28 560-579 42 42 42 81 81 81 F F F 29 580-599 43 43 43 82 82 82 F F F Max gen: 46, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 38 38 38 75 75 75 F F F 2 040-059 46 46 46 85 85 85 F F F 3 060-079 46 46 46 85 85 85 F F F 4 080-099 43 43 43 82 82 82 F F F 5 100-119 46 46 46 85 85 85 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 47 47 47 86 86 86 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 47 47 47 86 86 86 F F F 11 220-239 47 47 47 86 86 86 F F F 12 240-259 44 44 44 83 83 83 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 46 46 46 84 84 84 F F F 15 300-319 44 44 44 83 83 83 F F F 16 320-339 46 46 46 85 85 85 F F F 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 44 44 44 83 83 83 F F F 21 420-439 47 47 47 86 86 86 F F F 22 440-459 40 40 40 78 78 78 F F F 23 460-479 44 44 44 83 83 83 F F F 24 480-499 44 44 44 83 83 83 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 42 42 81 81 81 F F F 28 560-579 42 42 42 81 81 81 F F F 29 580-599 43 43 43 82 82 82 F F F Max gen: 47, Max flush: 86 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.136929231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.181877928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.182774104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.182824282Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.182840651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.182862391Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} {"msg":"Crucible 458be88a-6bc7-42de-a505-fca29b414b95 has session id: 3a9afa28-ffd2-48ac-a774-90f6bfcb514d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183431549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Upstairs opts: Upstairs UUID: 458be88a-6bc7-42de-a505-fca29b414b95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183496037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Crucible stats registered with UUID: 458be88a-6bc7-42de-a505-fca29b414b95","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183530905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:19.183557574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183734957Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183773536Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183795625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183968719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.183991408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} {"msg":"458be88a-6bc7-42de-a505-fca29b414b95 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184070525Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184091794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"0","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184109973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"1","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184126562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"2","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184142982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184160251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184177161Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37127)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184305926Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57067)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184351714Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51934)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.184387773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.185458832Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"1","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.185569938Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"0","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.186007541Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.186130506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.187774594Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"2","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[0]R flush_numbers[0..12]: [80, 75, 85, 85, 82, 85, 85, 86, 82, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188377691Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[0]R generation[0..12]: [42, 38, 46, 46, 43, 46, 46, 47, 43, 47, 47, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.18840235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188427409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[1]R flush_numbers[0..12]: [80, 75, 85, 85, 82, 85, 85, 86, 82, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188446138Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[1]R generation[0..12]: [42, 38, 46, 46, 43, 46, 46, 47, 43, 47, 47, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188464337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188483616Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[2]R flush_numbers[0..12]: [80, 75, 85, 85, 82, 85, 85, 86, 82, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188501866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[2]R generation[0..12]: [42, 38, 46, 46, 43, 46, 46, 47, 43, 47, 47, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188519985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188539274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188555254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188571893Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188587613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188606612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"":"downstairs","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:58:19.188623791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188644041Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"0","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.18866082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"1","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188677439Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"client":"2","":"downstairs","session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188694348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188710068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"458be88a-6bc7-42de-a505-fca29b414b95 is now active with session: 3a9afa28-ffd2-48ac-a774-90f6bfcb514d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188727897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188743097Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398,"session_id":"3a9afa28-ffd2-48ac-a774-90f6bfcb514d"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188758966Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} {"msg":"Activated sub_volume 458be88a-6bc7-42de-a505-fca29b414b95","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:19.188772876Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 4096 data: 6 02/30 Write block 271 len 24576 data: 5 5 4 3 3 4 03/30 Read block 22 len 16384 04/30 Read block 216 len 24576 05/30 Flush 06/30 Write block 293 len 4096 data: 5 07/30 Read block 280 len 24576 08/30 Write block 253 len 24576 data: 8 7 7 7 7 7 09/30 Flush 10/30 Write block 451 len 4096 data: 4 11/30 Write block 299 len 36864 data: 7 8 9 8 5 4 5 3 4 12/30 Write block 406 len 8192 data: 6 6 13/30 Write block 99 len 36864 data: 2 3 3 3 3 3 2 2 2 14/30 Write block 97 len 28672 data: 4 3 3 4 4 4 4 15/30 Write block 331 len 8192 data: 2 2 16/30 Write block 29 len 16384 data: 6 5 3 4 17/30 Read block 171 len 16384 18/30 Read block 210 len 12288 19/30 Flush 20/30 Write block 159 len 4096 data: 4 21/30 Flush 22/30 Write block 551 len 28672 data: 4 4 4 4 3 3 5 23/30 Write block 412 len 20480 data: 4 5 5 4 3 24/30 Write block 336 len 8192 data: 5 7 25/30 Read block 192 len 4096 26/30 Write block 60 len 24576 data: 9 8 8 7 7 6 27/30 Read block 529 len 36864 28/30 Write block 591 len 12288 data: 4 5 5 29/30 Read block 550 len 32768 30/30 Write block 210 len 36864 data: 3 3 3 2 2 3 3 4 4 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:19.205223169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1398} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 48 48 38 89 89 75 F F F <--- 2 040-059 46 46 46 85 85 85 F F F 3 060-079 48 48 46 87 87 85 T T F <--- 4 080-099 48 48 43 89 89 82 F F F <--- 5 100-119 48 48 46 89 89 85 F F F <--- 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 47 90 90 86 F F F <--- 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 47 47 47 86 86 86 T T F <--- 11 220-239 47 47 47 86 86 86 F F F 12 240-259 48 48 44 88 88 83 F F F <--- 13 260-279 48 48 38 87 87 75 F F F <--- 14 280-299 48 48 46 89 89 84 F F F <--- 15 300-319 48 48 44 89 89 83 F F F <--- 16 320-339 48 48 46 89 89 85 T T F <--- 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 48 48 44 89 89 83 T T F <--- 21 420-439 47 47 47 86 86 86 F F F 22 440-459 48 48 40 89 89 78 F F F <--- 23 460-479 44 44 44 83 83 83 F F F 24 480-499 44 44 44 83 83 83 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 42 42 42 81 81 81 T T F <--- 28 560-579 42 42 42 81 81 81 F F F 29 580-599 43 43 43 82 82 82 T T F <--- Max gen: 48, Max flush: 90 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 48 48 48 89 89 89 F F F 2 040-059 46 46 46 85 85 85 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 48 48 48 89 89 89 F F F 5 100-119 48 48 48 89 89 89 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 49 49 49 91 91 91 F F F 11 220-239 47 47 47 86 86 86 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 87 87 87 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 48 48 48 89 89 89 F F F 16 320-339 49 49 49 91 91 91 F F F 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 47 47 47 86 86 86 F F F 22 440-459 48 48 48 89 89 89 F F F 23 460-479 44 44 44 83 83 83 F F F 24 480-499 44 44 44 83 83 83 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 49 49 49 91 91 91 F F F 28 560-579 42 42 42 81 81 81 F F F 29 580-599 49 49 49 91 91 91 F F F Max gen: 49, Max flush: 91 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.629831841Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.674838787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.6755534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.675603228Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.675620097Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.675643406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} {"msg":"Crucible 60845ba1-2df5-40ac-a01e-374cbb0e13bd has session id: c1a44b13-b364-4172-8193-12c3d7820ad1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676185625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Upstairs opts: Upstairs UUID: 60845ba1-2df5-40ac-a01e-374cbb0e13bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676254923Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Crucible stats registered with UUID: 60845ba1-2df5-40ac-a01e-374cbb0e13bd","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676290841Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:23.67632445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676446996Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676490194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676512773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676706756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676733905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} {"msg":"60845ba1-2df5-40ac-a01e-374cbb0e13bd active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676794672Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676817552Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"0","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676838771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"1","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.67685928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676892229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.676963636Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.677005234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:23.677100011Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42269)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.677219576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:23.677266395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:23.677308183Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"ds_connection connected from Ok(127.0.0.1:65367)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.677340781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.67737368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.677407179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.678397191Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"1","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.67843248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"0","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.679045007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:23.679086165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.678111113Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53807)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.678200599Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.67976129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[0]R flush_numbers[0..12]: [80, 89, 85, 91, 89, 89, 85, 90, 82, 86, 91, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680396066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[0]R generation[0..12]: [42, 48, 46, 49, 48, 48, 46, 48, 43, 47, 49, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680422714Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680443104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[1]R flush_numbers[0..12]: [80, 89, 85, 91, 89, 89, 85, 90, 82, 86, 91, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680461653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[1]R generation[0..12]: [42, 48, 46, 49, 48, 48, 46, 48, 43, 47, 49, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680479822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680499211Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[2]R flush_numbers[0..12]: [80, 89, 85, 91, 89, 89, 85, 90, 82, 86, 91, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680517431Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[2]R generation[0..12]: [42, 48, 46, 49, 48, 48, 46, 48, 43, 47, 49, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.68053544Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680554889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680570979Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680587638Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680603878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680625597Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"":"downstairs","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680647316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680664345Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"0","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680682165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"1","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680699794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"client":"2","":"downstairs","session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680718063Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680734802Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"60845ba1-2df5-40ac-a01e-374cbb0e13bd is now active with session: c1a44b13-b364-4172-8193-12c3d7820ad1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680753472Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680769321Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406,"session_id":"c1a44b13-b364-4172-8193-12c3d7820ad1"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.680785951Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} {"msg":"Activated sub_volume 60845ba1-2df5-40ac-a01e-374cbb0e13bd","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:33.68080033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 580 len 40960 data: 5 5 4 5 5 4 4 5 5 5 02/30 Flush 03/30 Write block 479 len 4096 data: 4 04/30 Read block 11 len 28672 05/30 Read block 426 len 12288 06/30 Read block 167 len 28672 07/30 Read block 156 len 36864 08/30 Write block 385 len 8192 data: 3 3 09/30 Write block 558 len 32768 data: 5 6 7 7 6 7 5 3 10/30 Read block 335 len 12288 11/30 Read block 546 len 24576 12/30 Write block 510 len 36864 data: 4 4 2 3 4 5 5 5 6 13/30 Write block 330 len 16384 data: 2 3 3 2 14/30 Read block 235 len 12288 15/30 Read block 205 len 12288 16/30 Flush 17/30 Write block 372 len 28672 data: 4 4 4 4 4 4 4 18/30 Write block 348 len 4096 data: 7 19/30 Read block 562 len 40960 20/30 Read block 297 len 28672 21/30 Write block 39 len 40960 data: 3 5 4 4 4 4 3 3 4 5 22/30 Write block 95 len 24576 data: 5 4 5 4 4 5 23/30 Flush 24/30 Write block 554 len 36864 data: 5 4 4 6 6 7 8 8 7 25/30 Read block 386 len 20480 26/30 Write block 358 len 4096 data: 5 27/30 Write block 431 len 8192 data: 8 6 28/30 Write block 503 len 24576 data: 9 9 9 8 5 4 29/30 Read block 489 len 20480 30/30 Read block 355 len 40960 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:33.700731341Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1406} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 50 48 94 94 89 F F F <--- 2 040-059 50 50 46 94 94 85 F F F <--- 3 060-079 49 49 49 91 91 91 F F F 4 080-099 50 50 48 94 94 89 F F F <--- 5 100-119 50 50 48 94 94 89 F F F <--- 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 49 49 49 91 91 91 F F F 11 220-239 47 47 47 86 86 86 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 87 87 87 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 48 48 48 89 89 89 F F F 16 320-339 50 50 49 93 93 91 F F F <--- 17 340-359 50 50 44 94 94 83 T T F <--- 18 360-379 50 50 42 94 94 80 F F F <--- 19 380-399 50 50 46 93 93 85 F F F <--- 20 400-419 49 49 49 91 91 91 F F F 21 420-439 47 47 47 86 86 86 T T F <--- 22 440-459 48 48 48 89 89 89 F F F 23 460-479 50 50 44 93 93 83 F F F <--- 24 480-499 44 44 44 83 83 83 F F F 25 500-519 50 50 47 93 93 86 T T F <--- 26 520-539 30 30 30 58 58 58 F F F 27 540-559 50 50 49 93 93 91 T T F <--- 28 560-579 50 50 42 93 93 81 T T F <--- 29 580-599 50 50 50 92 92 92 F F F Max gen: 50, Max flush: 94 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 50 50 94 94 94 F F F 2 040-059 50 50 50 94 94 94 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 49 49 49 91 91 91 F F F 11 220-239 47 47 47 86 86 86 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 48 48 87 87 87 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 48 48 48 89 89 89 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 51 51 51 95 95 95 F F F 18 360-379 50 50 50 94 94 94 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 48 48 48 89 89 89 F F F 23 460-479 50 50 50 93 93 93 F F F 24 480-499 44 44 44 83 83 83 F F F 25 500-519 51 51 51 95 95 95 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 51 51 51 95 95 95 F F F 28 560-579 51 51 51 95 95 95 F F F 29 580-599 50 50 50 92 92 92 F F F Max gen: 51, Max flush: 95 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.111656385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:38.155112729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.155542073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.186284632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.186937507Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.186983895Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.187002454Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.187037293Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} {"msg":"Crucible b783e997-fdb0-458c-b430-fe4bfb6bcd8e has session id: 5f409e01-c5cf-476f-88de-b2beb9e05990","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.18763286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Upstairs opts: Upstairs UUID: b783e997-fdb0-458c-b430-fe4bfb6bcd8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.187698978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Crucible stats registered with UUID: b783e997-fdb0-458c-b430-fe4bfb6bcd8e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.187722087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:38.187738836Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.18791277Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.187959018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.187979247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.1881544Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.18817699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} {"msg":"b783e997-fdb0-458c-b430-fe4bfb6bcd8e active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188248067Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188268316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"0","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188285826Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"1","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188302595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"2","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188325364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188359283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188392491Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"ds_connection connected from Ok(127.0.0.1:55206)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188441089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"ds_connection connected from Ok(127.0.0.1:51086)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188472648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"ds_connection connected from Ok(127.0.0.1:38077)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.188502857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.189985301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"1","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190024629Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"0","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190061398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"2","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190677624Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190704673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[0]R flush_numbers[0..12]: [80, 94, 94, 91, 94, 94, 85, 90, 82, 86, 91, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190733242Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[0]R generation[0..12]: [42, 50, 50, 49, 50, 50, 46, 48, 43, 47, 49, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190752552Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190772311Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[1]R flush_numbers[0..12]: [80, 94, 94, 91, 94, 94, 85, 90, 82, 86, 91, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.19079079Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[1]R generation[0..12]: [42, 50, 50, 49, 50, 50, 46, 48, 43, 47, 49, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190809199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190833828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[2]R flush_numbers[0..12]: [80, 94, 94, 91, 94, 94, 85, 90, 82, 86, 91, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190852808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[2]R generation[0..12]: [42, 50, 50, 49, 50, 50, 46, 48, 43, 47, 49, 47]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190870967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190890406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190906866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190923805Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190939714Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190955834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"":"downstairs","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:58:38.190972853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.190994432Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"0","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.191012022Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"1","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.191028871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"client":"2","":"downstairs","session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.19104602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.19106209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"b783e997-fdb0-458c-b430-fe4bfb6bcd8e is now active with session: 5f409e01-c5cf-476f-88de-b2beb9e05990","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.191079899Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.191097208Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414,"session_id":"5f409e01-c5cf-476f-88de-b2beb9e05990"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.191113158Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} {"msg":"Activated sub_volume b783e997-fdb0-458c-b430-fe4bfb6bcd8e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:38.191127027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 40960 data: 8 8 7 6 5 4 4 5 5 4 02/30 Write block 573 len 12288 data: 6 5 6 03/30 Write block 202 len 16384 data: 5 5 7 8 04/30 Read block 209 len 20480 05/30 Write block 455 len 8192 data: 3 4 06/30 Read block 444 len 20480 07/30 Flush 08/30 Write block 46 len 36864 data: 4 5 6 6 5 6 6 7 7 09/30 Write block 43 len 40960 data: 5 5 4 5 6 7 7 6 7 7 10/30 Read block 99 len 40960 11/30 Read block 180 len 20480 12/30 Write block 343 len 4096 data: 5 13/30 Read block 112 len 28672 14/30 Write block 261 len 12288 data: 6 5 4 15/30 Write block 234 len 16384 data: 3 4 5 6 16/30 Read block 52 len 16384 17/30 Read block 536 len 40960 18/30 Write block 222 len 8192 data: 6 7 19/30 Read block 563 len 8192 20/30 Read block 518 len 32768 21/30 Read block 513 len 16384 22/30 Read block 474 len 28672 23/30 Read block 63 len 16384 24/30 Flush 25/30 Write block 486 len 12288 data: 6 8 10 26/30 Flush 27/30 Write block 111 len 36864 data: 3 5 4 4 4 6 7 7 7 28/30 Write block 48 len 28672 data: 8 8 7 8 8 8 8 29/30 Write block 269 len 28672 data: 6 5 6 6 5 4 4 30/30 Write block 513 len 24576 data: 4 5 6 6 6 7 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 7 Sent Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 3 31 Done 4 28 27 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:38.20970619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1414} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 50 50 94 94 94 F F F 2 040-059 50 52 52 94 97 97 F T T <--- 3 060-079 49 49 49 91 91 91 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 50 50 50 94 94 94 F T T <--- 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 49 52 52 91 96 96 T F F <--- 11 220-239 47 52 52 86 97 97 F F F <--- 12 240-259 48 48 48 88 88 88 F F F 13 260-279 48 52 52 87 97 97 F T T <--- 14 280-299 48 48 48 89 89 89 F F F 15 300-319 48 48 48 89 89 89 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 51 52 52 95 97 97 F F F <--- 18 360-379 50 50 50 94 94 94 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 48 52 52 89 96 96 F F F <--- 23 460-479 50 50 50 93 93 93 F F F 24 480-499 44 52 52 83 98 98 T F F <--- 25 500-519 51 51 51 95 95 95 F T T <--- 26 520-539 30 30 30 58 58 58 F F F 27 540-559 51 51 51 95 95 95 F F F 28 560-579 51 52 52 95 96 96 T F F <--- 29 580-599 50 50 50 92 92 92 F F F Max gen: 52, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 50 50 94 94 94 F F F 2 040-059 53 53 53 99 99 99 F F F 3 060-079 49 49 49 91 91 91 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 52 52 52 96 96 96 F F F 11 220-239 52 52 52 97 97 97 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 53 53 53 99 99 99 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 48 48 48 89 89 89 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 50 50 50 94 94 94 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 52 52 52 96 96 96 F F F 23 460-479 50 50 50 93 93 93 F F F 24 480-499 52 52 52 98 98 98 F F F 25 500-519 53 53 53 99 99 99 F F F 26 520-539 30 30 30 58 58 58 F F F 27 540-559 51 51 51 95 95 95 F F F 28 560-579 52 52 52 96 96 96 F F F 29 580-599 50 50 50 92 92 92 F F F Max gen: 53, Max flush: 99 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.644579922Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:42.687181019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.687534426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.718316313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719063625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719124082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719141622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719164881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} {"msg":"Crucible 2bac3702-84c9-40ac-9f84-1da70c1c2ff1 has session id: 865299dd-1e96-40a8-9966-5cc40670476d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719695731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Upstairs opts: Upstairs UUID: 2bac3702-84c9-40ac-9f84-1da70c1c2ff1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719741519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Crucible stats registered with UUID: 2bac3702-84c9-40ac-9f84-1da70c1c2ff1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.719761268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:42.719777828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.71998054Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720024968Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720056727Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.72023045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.72025133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} {"msg":"2bac3702-84c9-40ac-9f84-1da70c1c2ff1 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720315867Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720352656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"0","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720379295Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"1","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720435412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"2","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720482011Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720521639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720560598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54532)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720599756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42899)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720643835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60977)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.720688613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.721912396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"1","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722028622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"0","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722057821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"2","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722526393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722624929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[0]R flush_numbers[0..12]: [80, 94, 99, 91, 94, 99, 85, 90, 82, 86, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722664428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[0]R generation[0..12]: [42, 50, 53, 49, 50, 53, 46, 48, 43, 47, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722684207Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722704086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[1]R flush_numbers[0..12]: [80, 94, 99, 91, 94, 99, 85, 90, 82, 86, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722722585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[1]R generation[0..12]: [42, 50, 53, 49, 50, 53, 46, 48, 43, 47, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722740545Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722759974Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[2]R flush_numbers[0..12]: [80, 94, 99, 91, 94, 99, 85, 90, 82, 86, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722778423Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[2]R generation[0..12]: [42, 50, 53, 49, 50, 53, 46, 48, 43, 47, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722796653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722815852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722831941Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722853551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Next flush: 100","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-02-14T23:58:42.722876559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722892979Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"":"downstairs","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722910028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722925128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"0","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722942087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"1","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722958926Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"client":"2","":"downstairs","session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722976036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.722992055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"2bac3702-84c9-40ac-9f84-1da70c1c2ff1 is now active with session: 865299dd-1e96-40a8-9966-5cc40670476d","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.723009954Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.723025144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422,"session_id":"865299dd-1e96-40a8-9966-5cc40670476d"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.723040973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} {"msg":"Activated sub_volume 2bac3702-84c9-40ac-9f84-1da70c1c2ff1","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:42.723054653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 4096 data: 4 02/30 Read block 111 len 28672 03/30 Read block 110 len 16384 04/30 Read block 388 len 4096 05/30 Write block 462 len 40960 data: 3 3 2 2 2 2 2 2 3 3 06/30 Write block 361 len 32768 data: 5 4 4 4 4 4 4 4 07/30 Read block 348 len 16384 08/30 Read block 346 len 12288 09/30 Read block 324 len 36864 10/30 Read block 230 len 4096 11/30 Write block 537 len 20480 data: 5 5 5 4 6 12/30 Write block 214 len 16384 data: 3 4 4 5 13/30 Write block 487 len 32768 data: 9 11 9 9 8 7 6 5 14/30 Write block 67 len 4096 data: 6 15/30 Read block 401 len 16384 16/30 Read block 91 len 16384 17/30 Read block 551 len 8192 18/30 Read block 410 len 36864 19/30 Read block 555 len 40960 20/30 Read block 56 len 40960 21/30 Read block 531 len 12288 22/30 Read block 500 len 24576 23/30 Read block 393 len 4096 24/30 Read block 568 len 8192 25/30 Write block 522 len 12288 data: 2 2 2 26/30 Write block 480 len 40960 data: 3 3 4 5 6 7 7 10 12 10 27/30 Read block 377 len 12288 28/30 Read block 437 len 28672 29/30 Write block 305 len 4096 data: 6 30/30 Read block 481 len 16384 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Read 4 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:42.741486771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1422} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 50 50 94 94 94 F F F 2 040-059 53 53 53 99 99 99 F F F 3 060-079 49 49 49 91 91 91 F T T <--- 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 52 52 52 96 96 96 F T T <--- 11 220-239 52 52 52 97 97 97 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 53 53 53 99 99 99 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 48 48 48 89 89 89 F T T <--- 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 50 50 50 94 94 94 F T T <--- 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 52 52 52 96 96 96 F F F 23 460-479 50 50 50 93 93 93 F T T <--- 24 480-499 52 52 52 98 98 98 F T T <--- 25 500-519 53 53 53 99 99 99 F F F 26 520-539 30 30 30 58 58 58 F T T <--- 27 540-559 51 51 51 95 95 95 F T T <--- 28 560-579 52 52 52 96 96 96 F F F 29 580-599 50 50 50 92 92 92 F F F Max gen: 53, Max flush: 99 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 50 50 94 94 94 F F F 2 040-059 53 53 53 99 99 99 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 48 48 90 90 90 F F F 8 160-179 43 43 43 82 82 82 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 52 52 52 97 97 97 F F F 12 240-259 48 48 48 88 88 88 F F F 13 260-279 53 53 53 99 99 99 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 54 54 54 100 100 100 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 54 54 54 100 100 100 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 52 52 52 96 96 96 F F F 23 460-479 54 54 54 100 100 100 F F F 24 480-499 54 54 54 100 100 100 F F F 25 500-519 53 53 53 99 99 99 F F F 26 520-539 54 54 54 100 100 100 F F F 27 540-559 54 54 54 100 100 100 F F F 28 560-579 52 52 52 96 96 96 F F F 29 580-599 50 50 50 92 92 92 F F F Max gen: 54, Max flush: 100 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.163365087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:47.213628733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.214150573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.245302066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.245973671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246019929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246040268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246062417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} {"msg":"Crucible ba1e02e3-3725-43b1-a4b5-f39741e2b0b3 has session id: 7f9b1d03-2293-4e2e-b210-f1d077ba6c43","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246566318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Upstairs opts: Upstairs UUID: ba1e02e3-3725-43b1-a4b5-f39741e2b0b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246620516Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Crucible stats registered with UUID: ba1e02e3-3725-43b1-a4b5-f39741e2b0b3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246654895Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:47.246675274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246858787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246898565Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.246918255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247095728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247116407Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} {"msg":"ba1e02e3-3725-43b1-a4b5-f39741e2b0b3 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247188474Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247207924Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"0","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247237872Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"1","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.24730687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"2","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247368458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247410906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247446184Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:53566)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.24756703Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:64896)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247624428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:50661)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.247667586Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.248668748Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"2","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.248703047Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"1","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.248859571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"0","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249309054Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249338283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[0]R flush_numbers[0..12]: [80, 94, 99, 100, 94, 99, 85, 90, 82, 86, 100, 97]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249465038Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[0]R generation[0..12]: [42, 50, 53, 54, 50, 53, 46, 48, 43, 47, 54, 52]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249489767Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249513016Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[1]R flush_numbers[0..12]: [80, 94, 99, 100, 94, 99, 85, 90, 82, 86, 100, 97]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249532095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[1]R generation[0..12]: [42, 50, 53, 54, 50, 53, 46, 48, 43, 47, 54, 52]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249550255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249570084Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[2]R flush_numbers[0..12]: [80, 94, 99, 100, 94, 99, 85, 90, 82, 86, 100, 97]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249588653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[2]R generation[0..12]: [42, 50, 53, 54, 50, 53, 46, 48, 43, 47, 54, 52]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249606762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249625881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Max found gen is 55","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249641901Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Generation requested: 56 >= found:55","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.24965852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Next flush: 101","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249681699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249697649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"":"downstairs","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249714418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249729478Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"0","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249746317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"1","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249762746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"client":"2","":"downstairs","session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249781946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249798045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"ba1e02e3-3725-43b1-a4b5-f39741e2b0b3 is now active with session: 7f9b1d03-2293-4e2e-b210-f1d077ba6c43","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249815564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-14T23:58:47.249830854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430,"session_id":"7f9b1d03-2293-4e2e-b210-f1d077ba6c43"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249850733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} {"msg":"Activated sub_volume ba1e02e3-3725-43b1-a4b5-f39741e2b0b3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:47.249864952Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 32768 data: 7 7 8 7 6 5 4 5 02/30 Read block 256 len 24576 03/30 Read block 358 len 36864 04/30 Flush 05/30 Write block 274 len 8192 data: 5 5 06/30 Write block 21 len 32768 data: 7 6 5 6 6 6 6 6 07/30 Write block 292 len 36864 data: 3 6 6 7 7 7 9 8 9 08/30 Write block 562 len 4096 data: 8 09/30 Read block 412 len 20480 10/30 Flush 11/30 Write block 445 len 12288 data: 8 9 8 12/30 Write block 301 len 32768 data: 10 9 6 5 7 4 5 5 13/30 Write block 558 len 36864 data: 7 8 9 9 9 8 6 4 3 14/30 Write block 159 len 20480 data: 5 5 7 6 5 15/30 Flush 16/30 Write block 304 len 32768 data: 6 8 5 6 6 7 8 8 17/30 Read block 239 len 32768 18/30 Read block 451 len 16384 19/30 Read block 420 len 12288 20/30 Write block 544 len 8192 data: 5 3 21/30 Write block 259 len 4096 data: 7 22/30 Read block 56 len 12288 23/30 Read block 591 len 36864 24/30 Read block 273 len 28672 25/30 Read block 237 len 12288 26/30 Write block 229 len 28672 data: 4 7 8 6 4 4 5 27/30 Read block 427 len 4096 28/30 Read block 301 len 28672 29/30 Write block 49 len 12288 data: 9 8 9 30/30 Read block 418 len 8192 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 1 26 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:47.26908413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1430} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 50 56 56 94 102 102 F F F <--- 2 040-059 53 53 53 99 99 99 F T T <--- 3 060-079 54 54 54 100 100 100 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 48 56 56 90 103 103 F F F <--- 8 160-179 43 56 56 82 103 103 F F F <--- 9 180-199 47 47 47 86 86 86 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 52 52 52 97 97 97 F T T <--- 12 240-259 48 48 48 88 88 88 F T T <--- 13 260-279 53 56 56 99 102 102 T F F <--- 14 280-299 48 56 56 89 102 102 F F F <--- 15 300-319 54 56 56 100 103 103 F T T <--- 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 54 54 54 100 100 100 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 56 56 56 101 103 103 F F F <--- 23 460-479 54 54 54 100 100 100 F F F 24 480-499 54 54 54 100 100 100 F F F 25 500-519 53 53 53 99 99 99 F F F 26 520-539 54 54 54 100 100 100 F F F 27 540-559 54 56 56 100 103 103 F T T <--- 28 560-579 52 56 56 96 103 103 F F F <--- 29 580-599 50 50 50 92 92 92 F F F Max gen: 56, Max flush: 103 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 56 56 56 102 102 102 F F F 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 46 46 46 85 85 85 F F F 7 140-159 56 56 56 103 103 103 F F F 8 160-179 56 56 56 103 103 103 F F F 9 180-199 47 47 47 86 86 86 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 57 57 57 104 104 104 F F F 12 240-259 57 57 57 104 104 104 F F F 13 260-279 56 56 56 102 102 102 F F F 14 280-299 56 56 56 102 102 102 F F F 15 300-319 57 57 57 104 104 104 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 54 54 54 100 100 100 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 49 49 91 91 91 F F F 21 420-439 51 51 51 95 95 95 F F F 22 440-459 56 56 56 103 103 103 F F F 23 460-479 54 54 54 100 100 100 F F F 24 480-499 54 54 54 100 100 100 F F F 25 500-519 53 53 53 99 99 99 F F F 26 520-539 54 54 54 100 100 100 F F F 27 540-559 57 57 57 104 104 104 F F F 28 560-579 56 56 56 103 103 103 F F F 29 580-599 50 50 50 92 92 92 F F F Max gen: 57, Max flush: 104 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.698462772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:51.742634859Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.743042294Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.774586112Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.775239058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.775292015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.775309575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.775334264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} {"msg":"Crucible aaee51fa-5567-4e75-8f21-1cec966eb012 has session id: 0871627f-f93e-4566-87e3-26d6bbbe556a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.775942751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Upstairs opts: Upstairs UUID: aaee51fa-5567-4e75-8f21-1cec966eb012, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776006148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Crucible stats registered with UUID: aaee51fa-5567-4e75-8f21-1cec966eb012","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776037777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:51.776061016Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776254279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776292277Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776311457Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776464231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.77648566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} {"msg":"aaee51fa-5567-4e75-8f21-1cec966eb012 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776553797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776576287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"0","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776594456Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"1","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776615795Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"2","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776644664Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776675983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776706371Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60835)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.77673921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40800)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776769969Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51948)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.776808878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778018772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"1","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778182125Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"2","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778227533Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"0","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778620639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778732534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[0]R flush_numbers[0..12]: [80, 102, 104, 100, 94, 99, 85, 103, 103, 86, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.7788518Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[0]R generation[0..12]: [42, 56, 57, 54, 50, 53, 46, 56, 56, 47, 54, 57]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778877809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778898218Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[1]R flush_numbers[0..12]: [80, 102, 104, 100, 94, 99, 85, 103, 103, 86, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778916917Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[1]R generation[0..12]: [42, 56, 57, 54, 50, 53, 46, 56, 56, 47, 54, 57]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778946246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.778982865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[2]R flush_numbers[0..12]: [80, 102, 104, 100, 94, 99, 85, 103, 103, 86, 100, 104]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779031513Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[2]R generation[0..12]: [42, 56, 57, 54, 50, 53, 46, 56, 56, 47, 54, 57]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779063022Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-14T23:58:51.779096791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779131809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779160468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779177297Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779197947Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"":"downstairs","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779226626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779255474Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"0","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779274364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"1","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779296843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"client":"2","":"downstairs","session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779326542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.77935683Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"aaee51fa-5567-4e75-8f21-1cec966eb012 is now active with session: 0871627f-f93e-4566-87e3-26d6bbbe556a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.77937779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779402739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1438,"session_id":"0871627f-f93e-4566-87e3-26d6bbbe556a"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779433148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} {"msg":"Activated sub_volume aaee51fa-5567-4e75-8f21-1cec966eb012","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:51.779455347Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 124 len 4096 data: 8 02/30 Read block 551 len 40960 03/30 Read block 233 len 28672 04/30 Write block 521 len 36864 data: 2 3 3 3 3 3 3 3 3 05/30 Write block 135 len 8192 data: 6 7 06/30 Write block 575 len 36864 data: 7 7 5 5 4 6 6 5 6 07/30 Write block 371 len 4096 data: 4 08/30 Read block 104 len 4096 09/30 Write block 505 len 4096 data: 10 10/30 Write block 188 len 20480 data: 5 5 4 4 2 11/30 Read block 214 len 8192 12/30 Write block 419 len 24576 data: 5 6 7 8 8 9 13/30 Write block 594 len 12288 data: 5 5 5 14/30 Read block 106 len 24576 15/30 Write block 500 len 20480 data: 5 5 8 10 10 16/30 Flush 17/30 Write block 456 len 40960 data: 5 3 3 3 3 3 4 4 3 3 18/30 Write block 123 len 32768 data: 8 9 8 7 7 8 5 5 19/30 Read block 394 len 4096 20/30 Flush 21/30 Write block 309 len 4096 data: 8 22/30 Read block 578 len 24576 23/30 Write block 165 len 16384 data: 4 4 5 5 24/30 Read block 591 len 32768 25/30 Read block 348 len 28672 26/30 Write block 485 len 40960 data: 8 8 11 13 11 10 9 8 7 6 27/30 Write block 593 len 28672 data: 6 6 6 6 5 4 4 28/30 Write block 224 len 20480 data: 7 4 3 4 4 29/30 Read block 451 len 16384 30/30 Write block 534 len 40960 data: 3 3 5 6 6 6 5 7 5 5 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 7 29 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:51.797323356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1438} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 56 56 56 102 102 102 F F F 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 46 58 58 85 106 106 T F F <--- 7 140-159 56 56 56 103 103 103 F F F 8 160-179 56 56 56 103 103 103 F T T <--- 9 180-199 47 58 58 86 105 105 F F F <--- 10 200-219 54 54 54 100 100 100 F F F 11 220-239 57 57 57 104 104 104 F T T <--- 12 240-259 57 57 57 104 104 104 F F F 13 260-279 56 56 56 102 102 102 F F F 14 280-299 56 56 56 102 102 102 F F F 15 300-319 57 57 57 104 104 104 F T T <--- 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 54 58 58 100 105 105 T F F <--- 19 380-399 50 50 50 93 93 93 F F F 20 400-419 49 58 58 91 105 105 F F F <--- 21 420-439 51 58 58 95 105 105 F F F <--- 22 440-459 56 58 58 103 106 106 F F F <--- 23 460-479 54 58 58 100 106 106 F F F <--- 24 480-499 54 54 54 100 100 100 F T T <--- 25 500-519 53 58 58 99 105 105 F F F <--- 26 520-539 54 58 58 100 105 105 T T T <--- 27 540-559 57 57 57 104 104 104 F T T <--- 28 560-579 56 58 58 103 105 105 T F F <--- 29 580-599 50 58 58 92 105 105 T T T <--- Max gen: 58, Max flush: 106 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 56 56 56 102 102 102 F F F 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 50 50 50 94 94 94 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 58 58 58 106 106 106 F F F 7 140-159 56 56 56 103 103 103 F F F 8 160-179 59 59 59 107 107 107 F F F 9 180-199 58 58 58 105 105 105 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 59 59 59 107 107 107 F F F 12 240-259 57 57 57 104 104 104 F F F 13 260-279 56 56 56 102 102 102 F F F 14 280-299 56 56 56 102 102 102 F F F 15 300-319 59 59 59 107 107 107 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 58 58 58 105 105 105 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 58 58 58 105 105 105 F F F 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 59 59 59 107 107 107 F F F 25 500-519 58 58 58 105 105 105 F F F 26 520-539 59 59 59 107 107 107 F F F 27 540-559 59 59 59 107 107 107 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 59 59 59 107 107 107 F F F Max gen: 59, Max flush: 107 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.252695937Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:56.296813837Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.297197222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.328058287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.328721741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.328769269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.328784739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.328806788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} {"msg":"Crucible 02873812-8016-4a69-897a-184f30322382 has session id: 4bf066fb-dd3f-4648-b954-5e294675ad4f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329356327Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Upstairs opts: Upstairs UUID: 02873812-8016-4a69-897a-184f30322382, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329394156Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Crucible stats registered with UUID: 02873812-8016-4a69-897a-184f30322382","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329412445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:56.329428515Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329621657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329654446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329672095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329869418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329891907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} {"msg":"02873812-8016-4a69-897a-184f30322382 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329967234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.329987643Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"0","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.330021492Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"1","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.330039131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"2","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.33005544Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.33007747Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.330105539Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46354)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.330173926Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39914)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.330386348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45953)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.330435206Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.331436458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"1","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.331527675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"0","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.331554483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"2","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332011096Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332125962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[0]R flush_numbers[0..12]: [80, 102, 104, 100, 94, 99, 106, 103, 107, 105, 100, 107]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332240237Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[0]R generation[0..12]: [42, 56, 57, 54, 50, 53, 58, 56, 59, 58, 54, 59]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332263807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332284126Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[1]R flush_numbers[0..12]: [80, 102, 104, 100, 94, 99, 106, 103, 107, 105, 100, 107]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332303265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[1]R generation[0..12]: [42, 56, 57, 54, 50, 53, 58, 56, 59, 58, 54, 59]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332321404Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332340824Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[2]R flush_numbers[0..12]: [80, 102, 104, 100, 94, 99, 106, 103, 107, 105, 100, 107]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332359773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[2]R generation[0..12]: [42, 56, 57, 54, 50, 53, 58, 56, 59, 58, 54, 59]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332378172Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332397561Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332413661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.33243042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Next flush: 108","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2026-02-14T23:58:56.332453319Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332469539Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"":"downstairs","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332486528Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332504717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"0","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332521887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"1","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332538746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"client":"2","":"downstairs","session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332555995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332571945Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"02873812-8016-4a69-897a-184f30322382 is now active with session: 4bf066fb-dd3f-4648-b954-5e294675ad4f","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332589684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332604883Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446,"session_id":"4bf066fb-dd3f-4648-b954-5e294675ad4f"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332620733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} {"msg":"Activated sub_volume 02873812-8016-4a69-897a-184f30322382","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:58:56.332634442Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 20480 data: 9 9 9 8 7 02/30 Flush 03/30 Write block 411 len 4096 data: 5 04/30 Write block 404 len 32768 data: 7 8 7 7 6 5 5 6 05/30 Read block 333 len 4096 06/30 Read block 246 len 40960 07/30 Write block 277 len 8192 data: 5 4 08/30 Read block 249 len 20480 09/30 Write block 302 len 24576 data: 10 7 7 9 6 7 10/30 Read block 101 len 32768 11/30 Read block 301 len 20480 12/30 Write block 567 len 24576 data: 4 5 7 7 7 6 13/30 Read block 430 len 36864 14/30 Read block 275 len 32768 15/30 Write block 372 len 28672 data: 5 5 5 5 5 5 5 16/30 Flush 17/30 Write block 295 len 40960 data: 8 8 8 10 9 10 11 11 8 8 18/30 Write block 529 len 40960 data: 4 2 2 2 2 4 4 6 7 7 19/30 Read block 516 len 12288 20/30 Write block 300 len 40960 data: 11 12 12 9 9 10 7 8 7 9 21/30 Read block 505 len 24576 22/30 Read block 146 len 8192 23/30 Read block 220 len 24576 24/30 Write block 318 len 20480 data: 5 6 6 6 6 25/30 Write block 529 len 28672 data: 5 3 3 3 3 5 5 26/30 Write block 94 len 8192 data: 4 6 27/30 Read block 134 len 32768 28/30 Read block 590 len 16384 29/30 Write block 585 len 12288 data: 5 5 6 30/30 Read block 393 len 20480 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 0 28 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:58:56.35265713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1446} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 56 56 56 102 102 102 F F F 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 50 50 50 94 94 94 F T T <--- 5 100-119 53 53 53 99 99 99 F F F 6 120-139 58 58 58 106 106 106 F F F 7 140-159 56 56 56 103 103 103 F F F 8 160-179 59 59 59 107 107 107 F F F 9 180-199 58 58 58 105 105 105 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 59 59 59 107 107 107 F F F 12 240-259 57 57 57 104 104 104 F F F 13 260-279 56 60 60 102 109 109 F F F <--- 14 280-299 56 56 56 102 102 102 F T T <--- 15 300-319 60 60 60 108 109 109 F T T <--- 16 320-339 50 50 50 93 93 93 F T T <--- 17 340-359 52 52 52 97 97 97 F F F 18 360-379 58 60 60 105 109 109 F F F <--- 19 380-399 50 50 50 93 93 93 F F F 20 400-419 58 60 60 105 109 109 F F F <--- 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 59 59 59 107 107 107 F F F 25 500-519 58 58 58 105 105 105 F F F 26 520-539 59 59 59 107 107 107 F T T <--- 27 540-559 59 59 59 107 107 107 F F F 28 560-579 58 60 60 105 109 109 F F F <--- 29 580-599 59 59 59 107 107 107 F T T <--- Max gen: 60, Max flush: 109 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 56 56 56 102 102 102 F F F 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 61 61 61 110 110 110 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 58 58 58 106 106 106 F F F 7 140-159 56 56 56 103 103 103 F F F 8 160-179 59 59 59 107 107 107 F F F 9 180-199 58 58 58 105 105 105 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 59 59 59 107 107 107 F F F 12 240-259 57 57 57 104 104 104 F F F 13 260-279 60 60 60 109 109 109 F F F 14 280-299 61 61 61 110 110 110 F F F 15 300-319 61 61 61 110 110 110 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 60 60 60 109 109 109 F F F 19 380-399 50 50 50 93 93 93 F F F 20 400-419 60 60 60 109 109 109 F F F 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 59 59 59 107 107 107 F F F 25 500-519 58 58 58 105 105 105 F F F 26 520-539 61 61 61 110 110 110 F F F 27 540-559 59 59 59 107 107 107 F F F 28 560-579 60 60 60 109 109 109 F F F 29 580-599 61 61 61 110 110 110 F F F Max gen: 61, Max flush: 110 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.791408384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.836038794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.836740477Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.836785965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.836802065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.836824074Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} {"msg":"Crucible 9df55573-0705-4d24-b7d3-766bcaa5498e has session id: 8ff03570-effa-46e0-bf7c-d213e2cdbb48","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.837398302Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Upstairs opts: Upstairs UUID: 9df55573-0705-4d24-b7d3-766bcaa5498e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.83744627Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Crucible stats registered with UUID: 9df55573-0705-4d24-b7d3-766bcaa5498e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.837470019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:00.837486589Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.83772226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.837764058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.837783887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.83798051Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838014139Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} {"msg":"9df55573-0705-4d24-b7d3-766bcaa5498e active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838041708Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838057447Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"0","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838074226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"1","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838099455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838129774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838162773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838182952Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"ds_connection connected from Ok(127.0.0.1:44596)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838356456Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"ds_connection connected from Ok(127.0.0.1:36414)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838404424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:00.838441253Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:00.838475071Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:00.838551368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838616576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.838649294Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.839721704Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"1","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.839754632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"0","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.840311831Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:00.84033942Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.839393766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"ds_connection connected from Ok(127.0.0.1:53445)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.839483733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.84114283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[0]R flush_numbers[0..12]: [80, 102, 104, 100, 110, 99, 106, 103, 107, 105, 100, 107]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841759956Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[0]R generation[0..12]: [42, 56, 57, 54, 61, 53, 58, 56, 59, 58, 54, 59]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841798045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841823694Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[1]R flush_numbers[0..12]: [80, 102, 104, 100, 110, 99, 106, 103, 107, 105, 100, 107]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841847613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[1]R generation[0..12]: [42, 56, 57, 54, 61, 53, 58, 56, 59, 58, 54, 59]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841870312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841894621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[2]R flush_numbers[0..12]: [80, 102, 104, 100, 110, 99, 106, 103, 107, 105, 100, 107]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.84191802Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[2]R generation[0..12]: [42, 56, 57, 54, 61, 53, 58, 56, 59, 58, 54, 59]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841940529Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841964618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.841984938Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842006037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","Wait for a query_work_queue command to finish before sending IO pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842031416Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842051295Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"":"downstairs","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842072854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842091764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"0","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842115403Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"1","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842136542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"client":"2","":"downstairs","session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842158211Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.84218153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"9df55573-0705-4d24-b7d3-766bcaa5498e is now active with session: 8ff03570-effa-46e0-bf7c-d213e2cdbb48","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842204059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842223239Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454,"session_id":"8ff03570-effa-46e0-bf7c-d213e2cdbb48"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842243038Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} {"msg":"Activated sub_volume 9df55573-0705-4d24-b7d3-766bcaa5498e","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:10.842260377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 24576 data: 6 6 5 6 6 5 02/30 Flush 03/30 Write block 489 len 32768 data: 12 11 10 9 8 7 5 6 04/30 Write block 147 len 8192 data: 3 5 05/30 Read block 257 len 24576 06/30 Write block 393 len 8192 data: 5 5 07/30 Write block 220 len 24576 data: 4 5 7 8 8 5 08/30 Write block 172 len 36864 data: 6 6 7 6 5 4 4 4 3 09/30 Write block 524 len 12288 data: 4 4 4 10/30 Read block 535 len 8192 11/30 Read block 421 len 36864 12/30 Read block 173 len 28672 13/30 Read block 237 len 24576 14/30 Read block 59 len 8192 15/30 Read block 139 len 40960 16/30 Flush 17/30 Write block 20 len 4096 data: 6 18/30 Read block 266 len 28672 19/30 Write block 490 len 12288 data: 12 11 10 20/30 Write block 578 len 8192 data: 6 5 21/30 Read block 187 len 28672 22/30 Write block 368 len 8192 data: 5 4 23/30 Read block 200 len 24576 24/30 Read block 89 len 20480 25/30 Read block 202 len 8192 26/30 Read block 522 len 12288 27/30 Read block 219 len 36864 28/30 Read block 349 len 16384 29/30 Read block 120 len 8192 30/30 Write block 20 len 28672 data: 7 8 7 6 7 7 7 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 22 24 Done 27 27 6 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:10.860605108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1454} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 56 56 56 102 102 102 T T F <--- 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 61 61 61 110 110 110 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 58 58 58 106 106 106 F F F 7 140-159 62 62 56 112 112 103 F F T <--- 8 160-179 62 62 59 112 112 107 F F T <--- 9 180-199 62 62 58 112 112 105 F F T <--- 10 200-219 54 54 54 100 100 100 F F F 11 220-239 62 62 59 112 112 107 F F T <--- 12 240-259 57 57 57 104 104 104 F F F 13 260-279 60 60 60 109 109 109 F F F 14 280-299 61 61 61 110 110 110 F F F 15 300-319 61 61 61 110 110 110 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 62 62 62 111 111 111 F F F 18 360-379 60 60 60 109 109 109 T T F <--- 19 380-399 62 62 50 112 112 93 F F T <--- 20 400-419 60 60 60 109 109 109 F F F 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 62 62 59 112 112 107 T T T <--- 25 500-519 58 58 58 105 105 105 F F F 26 520-539 62 62 61 112 112 110 F F F <--- 27 540-559 59 59 59 107 107 107 F F F 28 560-579 60 60 60 109 109 109 T T F <--- 29 580-599 61 61 61 110 110 110 F F F Max gen: 62, Max flush: 112 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 57 57 57 104 104 104 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 61 61 61 110 110 110 F F F 5 100-119 53 53 53 99 99 99 F F F 6 120-139 58 58 58 106 106 106 F F F 7 140-159 62 62 62 112 112 112 F F F 8 160-179 62 62 62 112 112 112 F F F 9 180-199 62 62 62 112 112 112 F F F 10 200-219 54 54 54 100 100 100 F F F 11 220-239 62 62 62 112 112 112 F F F 12 240-259 57 57 57 104 104 104 F F F 13 260-279 60 60 60 109 109 109 F F F 14 280-299 61 61 61 110 110 110 F F F 15 300-319 61 61 61 110 110 110 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 62 62 62 111 111 111 F F F 18 360-379 63 63 63 113 113 113 F F F 19 380-399 62 62 62 112 112 112 F F F 20 400-419 60 60 60 109 109 109 F F F 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 63 63 63 113 113 113 F F F 25 500-519 58 58 58 105 105 105 F F F 26 520-539 62 62 62 112 112 112 F F F 27 540-559 59 59 59 107 107 107 F F F 28 560-579 63 63 63 113 113 113 F F F 29 580-599 61 61 61 110 110 110 F F F Max gen: 63, Max flush: 113 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.25691742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.303788974Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.304748628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.304814325Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.304844024Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.304876033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} {"msg":"Crucible 83c9df8f-a89c-442a-8abc-533c93717412 has session id: 4d24c6de-0eb5-4de1-86de-f1e7fe62da5a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305310127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Upstairs opts: Upstairs UUID: 83c9df8f-a89c-442a-8abc-533c93717412, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305353835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Crucible stats registered with UUID: 83c9df8f-a89c-442a-8abc-533c93717412","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305376434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:15.305392793Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305591796Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305637644Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305657963Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305853396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305879005Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} {"msg":"83c9df8f-a89c-442a-8abc-533c93717412 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305947562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305968142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"0","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.305985311Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"1","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.30600234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"2","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.306092967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.306123045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.306141905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57248)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.306160384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57767)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.306178593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42458)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.306439124Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.307342409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"0","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.307459695Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"1","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.307761593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"2","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.307906637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.307948076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[0]R flush_numbers[0..12]: [80, 113, 104, 100, 110, 99, 106, 112, 112, 112, 100, 112]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308290673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[0]R generation[0..12]: [42, 63, 57, 54, 61, 53, 58, 62, 62, 62, 54, 62]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308320482Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308340791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[1]R flush_numbers[0..12]: [80, 113, 104, 100, 110, 99, 106, 112, 112, 112, 100, 112]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.30835965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[1]R generation[0..12]: [42, 63, 57, 54, 61, 53, 58, 62, 62, 62, 54, 62]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.30837803Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308397909Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[2]R flush_numbers[0..12]: [80, 113, 104, 100, 110, 99, 106, 112, 112, 112, 100, 112]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308416978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[2]R generation[0..12]: [42, 63, 57, 54, 61, 53, 58, 62, 62, 62, 54, 62]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308435248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308460567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308477506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308494405Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308517654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308533704Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"":"downstairs","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308550483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308565792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"0","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308582482Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"1","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308599031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"client":"2","":"downstairs","session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.30861575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.30863187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"83c9df8f-a89c-442a-8abc-533c93717412 is now active with session: 4d24c6de-0eb5-4de1-86de-f1e7fe62da5a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308649689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308664859Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462,"session_id":"4d24c6de-0eb5-4de1-86de-f1e7fe62da5a"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308680708Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} {"msg":"Activated sub_volume 83c9df8f-a89c-442a-8abc-533c93717412","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:15.308694718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 189 len 4096 data: 6 03/30 Read block 197 len 40960 04/30 Write block 212 len 28672 data: 4 3 4 5 5 6 5 05/30 Write block 483 len 20480 data: 6 7 9 9 12 06/30 Write block 502 len 8192 data: 9 11 07/30 Read block 338 len 24576 08/30 Read block 148 len 16384 09/30 Write block 414 len 24576 data: 6 5 4 5 5 6 10/30 Write block 550 len 40960 data: 4 5 5 5 6 5 5 7 8 9 11/30 Read block 433 len 32768 12/30 Read block 53 len 36864 13/30 Read block 228 len 32768 14/30 Read block 258 len 40960 15/30 Write block 91 len 20480 data: 3 4 4 5 7 16/30 Read block 301 len 20480 17/30 Read block 233 len 24576 18/30 Read block 132 len 24576 19/30 Write block 234 len 40960 data: 5 6 6 7 5 5 3 3 3 3 20/30 Write block 158 len 20480 data: 3 6 6 8 7 21/30 Write block 58 len 4096 data: 6 22/30 Write block 590 len 36864 data: 6 5 6 7 7 7 7 6 5 23/30 Read block 74 len 40960 24/30 Write block 100 len 40960 data: 6 5 5 5 4 3 3 3 2 2 25/30 Read block 136 len 12288 26/30 Flush 27/30 Write block 0 len 16384 data: 2 4 4 4 28/30 Write block 343 len 40960 data: 6 6 7 8 9 8 7 7 6 7 29/30 Write block 503 len 16384 data: 12 11 11 9 30/30 Write block 538 len 28672 data: 8 7 6 8 6 6 6 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 23 29 Done 26 26 6 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:15.328634058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1462} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 80 80 80 T T F <--- 1 020-039 63 63 63 113 113 113 F F F 2 040-059 64 64 57 115 115 104 F F F <--- 3 060-079 54 54 54 100 100 100 F F F 4 080-099 64 64 61 115 115 110 F F F <--- 5 100-119 64 64 53 115 115 99 F F F <--- 6 120-139 58 58 58 106 106 106 F F F 7 140-159 64 64 62 115 115 112 F F F <--- 8 160-179 64 64 62 115 115 112 F F F <--- 9 180-199 64 64 62 115 115 112 F F T <--- 10 200-219 64 64 54 115 115 100 F F T <--- 11 220-239 64 64 62 115 115 112 F F F <--- 12 240-259 64 64 57 115 115 104 F F F <--- 13 260-279 60 60 60 109 109 109 F F F 14 280-299 61 61 61 110 110 110 F F F 15 300-319 61 61 61 110 110 110 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 62 62 62 111 111 111 T T F <--- 18 360-379 63 63 63 113 113 113 F F F 19 380-399 62 62 62 112 112 112 F F F 20 400-419 64 64 60 115 115 109 F F F <--- 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 64 64 63 115 115 113 F F T <--- 25 500-519 64 64 58 115 115 105 T T T <--- 26 520-539 62 62 62 112 112 112 T T F <--- 27 540-559 64 64 59 115 115 107 T T F <--- 28 560-579 63 63 63 113 113 113 F F F 29 580-599 64 64 61 115 115 110 F F F <--- Max gen: 64, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 64 64 64 115 115 115 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 64 64 64 115 115 115 F F F 5 100-119 64 64 64 115 115 115 F F F 6 120-139 58 58 58 106 106 106 F F F 7 140-159 64 64 64 115 115 115 F F F 8 160-179 64 64 64 115 115 115 F F F 9 180-199 64 64 64 115 115 115 F F F 10 200-219 64 64 64 115 115 115 F F F 11 220-239 64 64 64 115 115 115 F F F 12 240-259 64 64 64 115 115 115 F F F 13 260-279 60 60 60 109 109 109 F F F 14 280-299 61 61 61 110 110 110 F F F 15 300-319 61 61 61 110 110 110 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 65 65 65 116 116 116 F F F 18 360-379 63 63 63 113 113 113 F F F 19 380-399 62 62 62 112 112 112 F F F 20 400-419 64 64 64 115 115 115 F F F 21 420-439 58 58 58 105 105 105 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 58 58 58 106 106 106 F F F 24 480-499 64 64 64 115 115 115 F F F 25 500-519 65 65 65 116 116 116 F F F 26 520-539 65 65 65 116 116 116 F F F 27 540-559 65 65 65 116 116 116 F F F 28 560-579 63 63 63 113 113 113 F F F 29 580-599 64 64 64 115 115 115 F F F Max gen: 65, Max flush: 116 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.774715283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.818984127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.819699199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.819747587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.819764777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.819786826Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} {"msg":"Crucible 04c60dd3-db23-4c0e-8acb-ace41d902a58 has session id: 8840ac3f-e68e-4471-a6f0-48c5cd537cff","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.820367504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Upstairs opts: Upstairs UUID: 04c60dd3-db23-4c0e-8acb-ace41d902a58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.820425802Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Crucible stats registered with UUID: 04c60dd3-db23-4c0e-8acb-ace41d902a58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.820448641Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:19.82046562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.820636194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.820677852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.820698131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821037648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821059648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} {"msg":"04c60dd3-db23-4c0e-8acb-ace41d902a58 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821137734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821161184Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"0","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821182893Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"1","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821211802Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"2","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821243021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821346986Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821390445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:61104)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.82152168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:39970)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821563808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:58750)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.821596487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.822530481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"0","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.822664396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"2","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823146288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823175597Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823267264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"1","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[0]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 115, 106, 115, 115, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823790493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[0]R generation[0..12]: [65, 63, 64, 54, 64, 64, 58, 64, 64, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823817923Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823838322Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[1]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 115, 106, 115, 115, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823857321Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[1]R generation[0..12]: [65, 63, 64, 54, 64, 64, 58, 64, 64, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.82388389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823904169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[2]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 115, 106, 115, 115, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823923008Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[2]R generation[0..12]: [65, 63, 64, 54, 64, 64, 58, 64, 64, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823941148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823960457Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.823976617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824012195Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824029924Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824045974Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"":"downstairs","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824062623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824077813Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"0","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-14T23:59:19.824096952Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"1","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824114721Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"client":"2","":"downstairs","session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.82413188Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.82414806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"04c60dd3-db23-4c0e-8acb-ace41d902a58 is now active with session: 8840ac3f-e68e-4471-a6f0-48c5cd537cff","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824166009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824181149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470,"session_id":"8840ac3f-e68e-4471-a6f0-48c5cd537cff"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824197048Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} {"msg":"Activated sub_volume 04c60dd3-db23-4c0e-8acb-ace41d902a58","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:19.824210768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 309 len 28672 data: 10 9 10 10 10 9 8 02/30 Write block 166 len 24576 data: 5 6 6 3 5 5 03/30 Flush 04/30 Write block 170 len 4096 data: 6 05/30 Read block 480 len 16384 06/30 Write block 427 len 16384 data: 7 6 7 6 07/30 Read block 526 len 8192 08/30 Read block 376 len 16384 09/30 Read block 571 len 24576 10/30 Read block 75 len 4096 11/30 Read block 32 len 12288 12/30 Read block 112 len 24576 13/30 Read block 83 len 16384 14/30 Write block 558 len 8192 data: 9 10 15/30 Write block 467 len 40960 data: 3 3 3 4 4 3 3 3 3 3 16/30 Flush 17/30 Write block 387 len 20480 data: 3 3 3 5 5 18/30 Write block 393 len 32768 data: 6 6 7 9 8 7 7 6 19/30 Read block 166 len 16384 20/30 Read block 394 len 28672 21/30 Write block 298 len 4096 data: 11 22/30 Write block 379 len 12288 data: 3 2 3 23/30 Read block 592 len 4096 24/30 Write block 117 len 36864 data: 8 8 8 8 8 8 9 10 9 25/30 Write block 235 len 36864 data: 7 7 8 6 6 4 4 4 4 26/30 Flush 27/30 Write block 529 len 36864 data: 6 4 4 4 4 6 6 7 8 28/30 Write block 426 len 8192 data: 6 8 29/30 Read block 173 len 16384 30/30 Write block 340 len 16384 data: 8 6 6 7 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 12 1 14 Done 13 2 13 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1015)) Some(JobId(1025)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:19.842797329Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1470} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 64 64 64 115 115 115 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 64 64 64 115 115 115 F F F 5 100-119 66 64 66 119 115 119 F F F <--- 6 120-139 66 58 66 119 106 119 F F F <--- 7 140-159 64 64 64 115 115 115 F F F 8 160-179 66 66 66 118 118 118 F F F 9 180-199 64 64 64 115 115 115 F F F 10 200-219 64 64 64 115 115 115 F F F 11 220-239 66 64 66 119 115 119 F F F <--- 12 240-259 66 64 66 119 115 119 F F F <--- 13 260-279 60 60 60 109 109 109 F F F 14 280-299 66 61 66 119 110 119 F F F <--- 15 300-319 66 66 66 117 117 117 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 65 65 65 116 116 116 T F T <--- 18 360-379 66 63 66 119 113 119 F F F <--- 19 380-399 66 62 66 119 112 119 F T F <--- 20 400-419 66 64 66 119 115 119 F T F <--- 21 420-439 66 66 66 118 118 118 T F T <--- 22 440-459 58 58 58 106 106 106 F F F 23 460-479 66 66 66 118 118 118 F F F 24 480-499 64 64 64 115 115 115 F F F 25 500-519 65 65 65 116 116 116 F F F 26 520-539 65 65 65 116 116 116 T F T <--- 27 540-559 66 66 66 118 118 118 F F F 28 560-579 63 63 63 113 113 113 F F F 29 580-599 64 64 64 115 115 115 F F F Max gen: 66, Max flush: 119 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 64 64 64 115 115 115 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 64 64 64 115 115 115 F F F 5 100-119 66 66 66 119 119 119 F F F 6 120-139 66 66 66 119 119 119 F F F 7 140-159 64 64 64 115 115 115 F F F 8 160-179 66 66 66 118 118 118 F F F 9 180-199 64 64 64 115 115 115 F F F 10 200-219 64 64 64 115 115 115 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 60 60 60 109 109 109 F F F 14 280-299 66 66 66 119 119 119 F F F 15 300-319 66 66 66 117 117 117 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 67 67 67 120 120 120 F F F 18 360-379 66 66 66 119 119 119 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 67 67 67 120 120 120 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 66 66 66 118 118 118 F F F 24 480-499 64 64 64 115 115 115 F F F 25 500-519 65 65 65 116 116 116 F F F 26 520-539 67 67 67 120 120 120 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 63 63 63 113 113 113 F F F 29 580-599 64 64 64 115 115 115 F F F Max gen: 67, Max flush: 120 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.240078512Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.284342366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285066159Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285114177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285130266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285152295Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} {"msg":"Crucible b44791b1-76f0-4863-9f50-cbcf6d61d805 has session id: a2e38b05-3a2b-44e8-b192-476fd1b4b4dc","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285639297Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Upstairs opts: Upstairs UUID: b44791b1-76f0-4863-9f50-cbcf6d61d805, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285697575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Crucible stats registered with UUID: b44791b1-76f0-4863-9f50-cbcf6d61d805","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285723694Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:24.285744173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.285943745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286013183Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286044751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286170827Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286199216Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} {"msg":"b44791b1-76f0-4863-9f50-cbcf6d61d805 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286253874Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286283312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"0","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286302221Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"1","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286319371Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.28633725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286366729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286395378Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:24.286431137Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:46040)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286464286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35664)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286497664Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:24.286537123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:24.286586231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286658288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.286700337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.287712198Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"1","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.287755906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"0","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.288322635Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:24.288357453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.287119351Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:55002)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.287210307Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.288921372Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[0]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 119, 119, 115, 118, 115, 115, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289581797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[0]R generation[0..12]: [65, 63, 64, 54, 64, 66, 66, 64, 66, 64, 64, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289612605Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289633685Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[1]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 119, 119, 115, 118, 115, 115, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289653044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[1]R generation[0..12]: [65, 63, 64, 54, 64, 66, 66, 64, 66, 64, 64, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289676853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289696782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[2]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 119, 119, 115, 118, 115, 115, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289715472Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[2]R generation[0..12]: [65, 63, 64, 54, 64, 66, 66, 64, 66, 64, 64, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289733571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.28975316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.28976965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289786189Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289802108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289818128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"":"downstairs","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289834927Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289850147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"0","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289867006Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"1","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289889765Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"client":"2","":"downstairs","session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289906904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289922654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"b44791b1-76f0-4863-9f50-cbcf6d61d805 is now active with session: a2e38b05-3a2b-44e8-b192-476fd1b4b4dc","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289942683Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289958072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478,"session_id":"a2e38b05-3a2b-44e8-b192-476fd1b4b4dc"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289974042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} {"msg":"Activated sub_volume b44791b1-76f0-4863-9f50-cbcf6d61d805","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:34.289987911Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 433 len 12288 data: 5 7 8 02/30 Read block 215 len 16384 03/30 Read block 534 len 40960 04/30 Write block 170 len 28672 data: 7 6 7 7 8 7 6 05/30 Write block 570 len 8192 data: 8 8 06/30 Write block 312 len 12288 data: 11 11 10 07/30 Read block 328 len 32768 08/30 Write block 497 len 8192 data: 6 5 09/30 Write block 495 len 36864 data: 6 7 7 6 4 6 6 10 13 10/30 Write block 363 len 4096 data: 5 11/30 Write block 267 len 36864 data: 4 5 7 6 7 7 6 6 6 12/30 Write block 201 len 32768 data: 6 6 6 8 9 7 6 6 13/30 Read block 455 len 40960 14/30 Read block 357 len 32768 15/30 Flush 16/30 Write block 360 len 36864 data: 5 6 5 6 5 5 5 5 6 17/30 Read block 559 len 28672 18/30 Write block 151 len 36864 data: 4 5 6 4 4 4 3 4 7 19/30 Read block 70 len 32768 20/30 Flush 21/30 Write block 524 len 8192 data: 5 5 22/30 Write block 138 len 36864 data: 7 6 5 5 5 3 2 3 3 23/30 Read block 236 len 32768 24/30 Write block 342 len 4096 data: 7 25/30 Read block 480 len 20480 26/30 Read block 586 len 16384 27/30 Read block 447 len 36864 28/30 Write block 539 len 36864 data: 8 7 9 7 7 7 4 3 2 29/30 Read block 403 len 24576 30/30 Read block 35 len 28672 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 4 Done Done Sent false 1002 Acked Read 10 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:34.310998441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1478} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 64 64 64 115 115 115 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 64 64 64 115 115 115 F F F 5 100-119 66 66 66 119 119 119 F F F 6 120-139 66 66 66 119 119 119 T T F <--- 7 140-159 68 68 64 122 122 115 T T F <--- 8 160-179 68 68 66 121 121 118 F F F <--- 9 180-199 64 64 64 115 115 115 F F F 10 200-219 68 68 64 121 121 115 F F F <--- 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 68 68 60 121 121 109 F F F <--- 14 280-299 66 66 66 119 119 119 F F F 15 300-319 68 68 66 121 121 117 F F F <--- 16 320-339 61 61 61 110 110 110 F F F 17 340-359 67 67 67 120 120 120 T T F <--- 18 360-379 68 68 66 122 122 119 F F F <--- 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 68 68 67 121 121 120 F F T <--- 22 440-459 58 58 58 106 106 106 F F F 23 460-479 66 66 66 118 118 118 F F F 24 480-499 68 68 64 121 121 115 F F F <--- 25 500-519 68 68 65 121 121 116 F F F <--- 26 520-539 67 67 67 120 120 120 T T F <--- 27 540-559 66 66 66 118 118 118 T T F <--- 28 560-579 68 68 63 121 121 113 F F F <--- 29 580-599 64 64 64 115 115 115 F F F Max gen: 68, Max flush: 122 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 64 64 64 115 115 115 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 64 64 64 115 115 115 F F F 5 100-119 66 66 66 119 119 119 F F F 6 120-139 69 69 69 123 123 123 F F F 7 140-159 69 69 69 123 123 123 F F F 8 160-179 68 68 68 121 121 121 F F F 9 180-199 64 64 64 115 115 115 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 66 66 66 119 119 119 F F F 15 300-319 68 68 68 121 121 121 F F F 16 320-339 61 61 61 110 110 110 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 68 68 68 122 122 122 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 68 68 68 121 121 121 F F F 22 440-459 58 58 58 106 106 106 F F F 23 460-479 66 66 66 118 118 118 F F F 24 480-499 68 68 68 121 121 121 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 69 69 69 123 123 123 F F F 27 540-559 69 69 69 123 123 123 F F F 28 560-579 68 68 68 121 121 121 F F F 29 580-599 64 64 64 115 115 115 F F F Max gen: 69, Max flush: 123 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.730061256Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.775474606Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.776247626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.776295665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.776312914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.776334743Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} {"msg":"Crucible 7325fe43-d9c3-46ae-ba74-223dbdf5f4fa has session id: 9df0a607-0593-48ce-990a-43bf0a1a93c6","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.776764156Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Upstairs opts: Upstairs UUID: 7325fe43-d9c3-46ae-ba74-223dbdf5f4fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.776899411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Crucible stats registered with UUID: 7325fe43-d9c3-46ae-ba74-223dbdf5f4fa","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.77694192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:38.776976368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.77720756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777248598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777267907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777508808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777529658Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} {"msg":"7325fe43-d9c3-46ae-ba74-223dbdf5f4fa active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777605604Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777626304Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"0","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777649323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"1","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777678722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"2","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.77771138Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777744129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777780298Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48658)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777823506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43003)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777862485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49659)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.777899533Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779198254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"0","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779237602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"2","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779260541Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"1","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779875218Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779904157Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[0]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 119, 123, 123, 121, 115, 121, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779927856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[0]R generation[0..12]: [65, 63, 64, 54, 64, 66, 69, 69, 68, 64, 68, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779946945Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779967034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[1]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 119, 123, 123, 121, 115, 121, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.779986044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[1]R generation[0..12]: [65, 63, 64, 54, 64, 66, 69, 69, 68, 64, 68, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780004293Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780023732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[2]R flush_numbers[0..12]: [116, 113, 115, 100, 115, 119, 123, 123, 121, 115, 121, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780042751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[2]R generation[0..12]: [65, 63, 64, 54, 64, 66, 69, 69, 68, 64, 68, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780061031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.7800806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780096849Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780117028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780133168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780149187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"":"downstairs","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780166177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780187446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"0","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780204055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"1","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780220905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"client":"2","":"downstairs","session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780237984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780253903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"7325fe43-d9c3-46ae-ba74-223dbdf5f4fa is now active with session: 9df0a607-0593-48ce-990a-43bf0a1a93c6","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780271753Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780286692Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486,"session_id":"9df0a607-0593-48ce-990a-43bf0a1a93c6"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780302721Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} {"msg":"Activated sub_volume 7325fe43-d9c3-46ae-ba74-223dbdf5f4fa","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:38.780316501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 12288 data: 8 9 8 02/30 Write block 213 len 12288 data: 4 5 6 03/30 Read block 407 len 36864 04/30 Read block 557 len 32768 05/30 Write block 198 len 16384 data: 6 6 8 7 06/30 Read block 345 len 4096 07/30 Write block 41 len 32768 data: 5 5 6 6 5 6 7 9 08/30 Write block 130 len 24576 data: 6 5 5 4 4 7 09/30 Write block 329 len 36864 data: 2 3 4 4 3 3 4 6 8 10/30 Read block 245 len 12288 11/30 Flush 12/30 Write block 82 len 32768 data: 4 6 6 5 5 4 5 5 13/30 Read block 39 len 28672 14/30 Flush 15/30 Write block 317 len 36864 data: 5 6 7 7 7 7 6 3 3 16/30 Read block 122 len 16384 17/30 Write block 552 len 32768 data: 6 6 7 6 6 8 10 11 18/30 Flush 19/30 Flush 20/30 Write block 82 len 12288 data: 5 7 7 21/30 Write block 128 len 28672 data: 9 6 7 6 6 5 5 22/30 Write block 160 len 24576 data: 7 9 8 6 5 5 23/30 Flush 24/30 Write block 311 len 36864 data: 11 12 12 11 9 5 6 7 8 25/30 Write block 548 len 16384 data: 3 4 5 6 26/30 Read block 308 len 12288 27/30 Read block 526 len 36864 28/30 Write block 432 len 40960 data: 7 6 8 9 8 7 7 7 7 6 29/30 Write block 47 len 4096 data: 8 30/30 Write block 72 len 8192 data: 7 8 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 30 3 36 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:38.799687843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1486} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 70 64 70 124 115 124 T F T <--- 3 060-079 54 54 54 100 100 100 T F T <--- 4 080-099 70 64 70 128 115 128 F F F <--- 5 100-119 66 66 66 119 119 119 F F F 6 120-139 70 69 70 128 123 128 F F F <--- 7 140-159 69 69 69 123 123 123 F F F 8 160-179 70 68 70 128 121 128 F F F <--- 9 180-199 70 64 70 124 115 124 F F F <--- 10 200-219 70 68 70 124 121 124 F F F <--- 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 66 66 66 119 119 119 F F F 15 300-319 70 68 70 126 121 126 T F T <--- 16 320-339 70 61 70 126 110 126 F F F <--- 17 340-359 69 69 69 123 123 123 F F F 18 360-379 68 68 68 122 122 122 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 68 68 68 121 121 121 T F T <--- 22 440-459 58 58 58 106 106 106 T F T <--- 23 460-479 66 66 66 118 118 118 F F F 24 480-499 68 68 68 121 121 121 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 69 69 69 123 123 123 F F F 27 540-559 70 69 70 126 123 126 T F T <--- 28 560-579 68 68 68 121 121 121 F F F 29 580-599 64 64 64 115 115 115 F F F Max gen: 70, Max flush: 128 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 63 63 63 113 113 113 F F F 2 040-059 71 71 71 129 129 129 F F F 3 060-079 71 71 71 129 129 129 F F F 4 080-099 70 70 70 128 128 128 F F F 5 100-119 66 66 66 119 119 119 F F F 6 120-139 70 70 70 128 128 128 F F F 7 140-159 69 69 69 123 123 123 F F F 8 160-179 70 70 70 128 128 128 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 66 66 66 119 119 119 F F F 15 300-319 71 71 71 129 129 129 F F F 16 320-339 70 70 70 126 126 126 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 68 68 68 122 122 122 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 71 71 71 129 129 129 F F F 22 440-459 71 71 71 129 129 129 F F F 23 460-479 66 66 66 118 118 118 F F F 24 480-499 68 68 68 121 121 121 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 69 69 69 123 123 123 F F F 27 540-559 71 71 71 129 129 129 F F F 28 560-579 68 68 68 121 121 121 F F F 29 580-599 64 64 64 115 115 115 F F F Max gen: 71, Max flush: 129 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.209325237Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.253758434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.254454798Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.254502686Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.254523035Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.254545204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} {"msg":"Crucible 1124223b-f8d9-410d-955a-e540293d6241 has session id: 489277fd-a8f2-42c7-9251-a11afa394c31","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255056995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Upstairs opts: Upstairs UUID: 1124223b-f8d9-410d-955a-e540293d6241, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255116323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Crucible stats registered with UUID: 1124223b-f8d9-410d-955a-e540293d6241","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255146281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:43.25517206Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255405272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.25545255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255472799Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255631423Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255678411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} {"msg":"1124223b-f8d9-410d-955a-e540293d6241 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255742479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255769508Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"0","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255787487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"1","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255809116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"2","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255836215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255866684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255899003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"ds_connection connected from Ok(127.0.0.1:64367)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255925432Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"ds_connection connected from Ok(127.0.0.1:51752)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.255951711Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"ds_connection connected from Ok(127.0.0.1:58417)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.25598152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.257164515Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"2","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.257210553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"0","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.257752932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.25780166Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.258885519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"1","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[0]R flush_numbers[0..12]: [116, 113, 129, 129, 128, 119, 128, 123, 128, 124, 124, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259403599Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[0]R generation[0..12]: [65, 63, 71, 71, 70, 66, 70, 69, 70, 70, 70, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259432298Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259457317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[1]R flush_numbers[0..12]: [116, 113, 129, 129, 128, 119, 128, 123, 128, 124, 124, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259476687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[1]R generation[0..12]: [65, 63, 71, 71, 70, 66, 70, 69, 70, 70, 70, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259494856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259514445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[2]R flush_numbers[0..12]: [116, 113, 129, 129, 128, 119, 128, 123, 128, 124, 124, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259533385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[2]R generation[0..12]: [65, 63, 71, 71, 70, 66, 70, 69, 70, 70, 70, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259551694Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259576823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259611962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.25964483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259676759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259708788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"":"downstairs","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259741817Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259772585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"0","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259799764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"1","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259829853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"client":"2","":"downstairs","session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2026-02-14T23:59:43.259870612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.25989792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"1124223b-f8d9-410d-955a-e540293d6241 is now active with session: 489277fd-a8f2-42c7-9251-a11afa394c31","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.25991671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259931899Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494,"session_id":"489277fd-a8f2-42c7-9251-a11afa394c31"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259947769Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} {"msg":"Activated sub_volume 1124223b-f8d9-410d-955a-e540293d6241","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:43.259961608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 20480 data: 7 8 10 10 13 02/30 Flush 03/30 Write block 97 len 12288 data: 6 5 5 04/30 Read block 334 len 32768 05/30 Flush 06/30 Write block 82 len 24576 data: 6 8 8 6 6 5 07/30 Flush 08/30 Write block 38 len 16384 data: 4 4 6 6 09/30 Write block 471 len 8192 data: 5 4 10/30 Write block 575 len 40960 data: 8 8 6 7 6 7 7 6 7 6 11/30 Read block 184 len 16384 12/30 Read block 252 len 16384 13/30 Write block 562 len 12288 data: 10 9 7 14/30 Read block 424 len 20480 15/30 Read block 475 len 12288 16/30 Read block 101 len 24576 17/30 Read block 32 len 8192 18/30 Flush 19/30 Write block 100 len 40960 data: 7 6 6 6 5 4 4 4 3 3 20/30 Flush 21/30 Write block 524 len 4096 data: 6 22/30 Write block 371 len 12288 data: 5 6 6 23/30 Write block 290 len 40960 data: 3 3 4 7 7 9 9 9 12 10 24/30 Read block 1 len 24576 25/30 Read block 269 len 4096 26/30 Write block 434 len 12288 data: 9 10 9 27/30 Write block 58 len 20480 data: 7 8 10 9 9 28/30 Read block 24 len 20480 29/30 Read block 335 len 4096 30/30 Read block 188 len 40960 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1001)) Some(JobId(1019)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:43.278500792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1494} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 72 63 72 133 113 133 F F F <--- 2 040-059 72 71 72 133 129 133 T F T <--- 3 060-079 71 71 71 129 129 129 T F T <--- 4 080-099 72 70 72 132 128 132 F F F <--- 5 100-119 72 66 72 134 119 134 F F F <--- 6 120-139 70 70 70 128 128 128 F F F 7 140-159 69 69 69 123 123 123 F F F 8 160-179 70 70 70 128 128 128 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 66 66 66 119 119 119 T F T <--- 15 300-319 71 71 71 129 129 129 F F F 16 320-339 70 70 70 126 126 126 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 68 68 68 122 122 122 T F T <--- 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 71 71 71 129 129 129 T F T <--- 22 440-459 71 71 71 129 129 129 F F F 23 460-479 72 66 72 133 118 133 F F F <--- 24 480-499 72 72 72 130 130 130 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 69 69 69 123 123 123 T F T <--- 27 540-559 71 71 71 129 129 129 F F F 28 560-579 72 68 72 133 121 133 F F F <--- 29 580-599 72 64 72 133 115 133 F F F <--- Max gen: 72, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 F F F 1 020-039 72 72 72 133 133 133 F F F 2 040-059 73 73 73 135 135 135 F F F 3 060-079 73 73 73 135 135 135 F F F 4 080-099 72 72 72 132 132 132 F F F 5 100-119 72 72 72 134 134 134 F F F 6 120-139 70 70 70 128 128 128 F F F 7 140-159 69 69 69 123 123 123 F F F 8 160-179 70 70 70 128 128 128 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 71 71 71 129 129 129 F F F 16 320-339 70 70 70 126 126 126 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 73 73 73 135 135 135 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 73 73 73 135 135 135 F F F 22 440-459 71 71 71 129 129 129 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 130 130 130 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 73 73 73 135 135 135 F F F 27 540-559 71 71 71 129 129 129 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 133 133 133 F F F Max gen: 73, Max flush: 135 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.683595728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.729300497Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.730449623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.730511981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.73053248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.730569059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} {"msg":"Crucible fdbf7ede-c1ab-455d-9151-871e95a3d538 has session id: 9614a154-961f-436e-b5d4-d80c4b748f14","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731151067Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Upstairs opts: Upstairs UUID: fdbf7ede-c1ab-455d-9151-871e95a3d538, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731216374Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Crucible stats registered with UUID: fdbf7ede-c1ab-455d-9151-871e95a3d538","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731248593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:47.731279542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731490724Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731540102Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731559551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731751094Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731785743Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} {"msg":"fdbf7ede-c1ab-455d-9151-871e95a3d538 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731844081Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731875559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"0","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731896408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"1","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731913748Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"2","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731930557Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731956266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.731983765Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"ds_connection connected from Ok(127.0.0.1:55118)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.732164108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"ds_connection connected from Ok(127.0.0.1:34696)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.732220096Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"ds_connection connected from Ok(127.0.0.1:61293)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.732257265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.733393062Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"0","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.733592954Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"2","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.733733829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"1","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734070766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734182381Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[0]R flush_numbers[0..12]: [116, 133, 135, 135, 132, 134, 128, 123, 128, 124, 124, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734456481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[0]R generation[0..12]: [65, 72, 73, 73, 72, 72, 70, 69, 70, 70, 70, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.73448018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734500539Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[1]R flush_numbers[0..12]: [116, 133, 135, 135, 132, 134, 128, 123, 128, 124, 124, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734519648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[1]R generation[0..12]: [65, 72, 73, 73, 72, 72, 70, 69, 70, 70, 70, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734537988Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734557487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[2]R flush_numbers[0..12]: [116, 133, 135, 135, 132, 134, 128, 123, 128, 124, 124, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734576066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[2]R generation[0..12]: [65, 72, 73, 73, 72, 72, 70, 69, 70, 70, 70, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734594086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734613595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734635944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734653543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734669503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734685722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734710661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734726001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"0","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.73474316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"1","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734759939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"client":"2","":"downstairs","session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734776869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734792758Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"fdbf7ede-c1ab-455d-9151-871e95a3d538 is now active with session: 9614a154-961f-436e-b5d4-d80c4b748f14","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734810648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734825927Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502,"session_id":"9614a154-961f-436e-b5d4-d80c4b748f14"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734841596Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} {"msg":"Activated sub_volume fdbf7ede-c1ab-455d-9151-871e95a3d538","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:47.734855266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 24576 data: 7 4 8 7 8 8 02/30 Flush 03/30 Flush 04/30 Write block 307 len 16384 data: 10 9 11 10 05/30 Read block 87 len 4096 06/30 Write block 344 len 4096 data: 7 07/30 Read block 177 len 8192 08/30 Read block 90 len 36864 09/30 Write block 24 len 40960 data: 8 8 8 7 7 7 6 4 5 3 10/30 Write block 78 len 16384 data: 7 5 5 4 11/30 Write block 52 len 16384 data: 9 9 9 6 12/30 Write block 472 len 12288 data: 5 4 4 13/30 Read block 501 len 20480 14/30 Write block 117 len 28672 data: 9 9 9 9 9 9 10 15/30 Write block 246 len 24576 data: 5 4 4 4 4 5 16/30 Read block 354 len 36864 17/30 Read block 436 len 16384 18/30 Write block 54 len 8192 data: 10 7 19/30 Write block 0 len 24576 data: 3 5 5 5 4 5 20/30 Read block 459 len 20480 21/30 Read block 561 len 32768 22/30 Read block 447 len 12288 23/30 Read block 420 len 8192 24/30 Write block 585 len 12288 data: 6 6 7 25/30 Write block 320 len 12288 data: 8 8 8 26/30 Write block 437 len 16384 data: 8 8 8 8 27/30 Read block 127 len 4096 28/30 Write block 210 len 12288 data: 4 4 5 29/30 Read block 319 len 36864 30/30 Read block 363 len 8192 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) None Some(JobId(1002)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:47.752853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1502} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 116 116 116 T F T <--- 1 020-039 72 72 72 133 133 133 T F T <--- 2 040-059 73 73 73 135 135 135 T F T <--- 3 060-079 73 73 73 135 135 135 T F T <--- 4 080-099 72 72 72 132 132 132 T F T <--- 5 100-119 72 72 72 134 134 134 T F T <--- 6 120-139 70 70 70 128 128 128 T F T <--- 7 140-159 69 69 69 123 123 123 F F F 8 160-179 74 70 74 136 128 136 F F F <--- 9 180-199 70 70 70 124 124 124 F F F 10 200-219 70 70 70 124 124 124 T F T <--- 11 220-239 66 66 66 119 119 119 F F F 12 240-259 66 66 66 119 119 119 T F T <--- 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 71 71 71 129 129 129 T F T <--- 16 320-339 70 70 70 126 126 126 T F T <--- 17 340-359 69 69 69 123 123 123 T F T <--- 18 360-379 73 73 73 135 135 135 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 73 73 73 135 135 135 T F T <--- 22 440-459 71 71 71 129 129 129 T F T <--- 23 460-479 72 72 72 133 133 133 T F T <--- 24 480-499 72 72 72 130 130 130 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 73 73 73 135 135 135 F F F 27 540-559 71 71 71 129 129 129 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 133 133 133 T F T <--- Max gen: 74, Max flush: 136 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 137 137 137 F F F 1 020-039 75 75 75 137 137 137 F F F 2 040-059 75 75 75 137 137 137 F F F 3 060-079 75 75 75 137 137 137 F F F 4 080-099 75 75 75 137 137 137 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 69 69 69 123 123 123 F F F 8 160-179 74 74 74 136 136 136 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 75 75 75 137 137 137 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 75 75 75 137 137 137 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 73 73 73 135 135 135 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 75 75 75 137 137 137 F F F 22 440-459 75 75 75 137 137 137 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 72 72 72 130 130 130 F F F 25 500-519 68 68 68 121 121 121 F F F 26 520-539 73 73 73 135 135 135 F F F 27 540-559 71 71 71 129 129 129 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 75 75 75 137 137 137 F F F Max gen: 75, Max flush: 137 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.199581991Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.24738243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.248349643Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.248402401Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.248420131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.248448299Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} {"msg":"Crucible 354b70e8-443e-4897-a314-5a4c6cc08ee9 has session id: 59d1ed42-3c8e-4c4c-a679-50decd5f1355","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249178072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Upstairs opts: Upstairs UUID: 354b70e8-443e-4897-a314-5a4c6cc08ee9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249256059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Crucible stats registered with UUID: 354b70e8-443e-4897-a314-5a4c6cc08ee9","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249284678Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:52.249304907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249499389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249547737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249577567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249766319Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249804888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} {"msg":"354b70e8-443e-4897-a314-5a4c6cc08ee9 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249839437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249868325Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"0","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249895144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"1","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249922423Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"2","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249949092Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.249980501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.250026569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"ds_connection connected from Ok(127.0.0.1:42080)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.250062688Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"ds_connection connected from Ok(127.0.0.1:52715)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.250094397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"ds_connection connected from Ok(127.0.0.1:51713)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.250123266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.251353829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"0","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.251388487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"1","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.252219526Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.252248795Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.252479446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"2","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 137, 137, 137, 137, 137, 123, 136, 124, 137, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253071083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[0]R generation[0..12]: [75, 75, 75, 75, 75, 75, 75, 69, 74, 70, 75, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253101412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253126691Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 137, 137, 137, 137, 137, 123, 136, 124, 137, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253146611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[1]R generation[0..12]: [75, 75, 75, 75, 75, 75, 75, 69, 74, 70, 75, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.25316487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253184589Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 137, 137, 137, 137, 137, 123, 136, 124, 137, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253203418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[2]R generation[0..12]: [75, 75, 75, 75, 75, 75, 75, 69, 74, 70, 75, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253221468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253240817Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253257286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253273976Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253289925Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253305834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"":"downstairs","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253322564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253337623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"0","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253354203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"1","":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253370682Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"client":"2",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253400711Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.25341682Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"354b70e8-443e-4897-a314-5a4c6cc08ee9 is now active with session: 59d1ed42-3c8e-4c4c-a679-50decd5f1355","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253434669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253450549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510,"session_id":"59d1ed42-3c8e-4c4c-a679-50decd5f1355"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253466408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} {"msg":"Activated sub_volume 354b70e8-443e-4897-a314-5a4c6cc08ee9","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:52.253480168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 328 len 24576 data: 2 3 4 5 5 4 02/30 Read block 483 len 40960 03/30 Read block 511 len 16384 04/30 Write block 200 len 40960 data: 9 8 7 7 9 10 8 7 7 6 05/30 Read block 345 len 20480 06/30 Flush 07/30 Write block 163 len 16384 data: 7 6 6 6 08/30 Flush 09/30 Write block 40 len 24576 data: 7 7 6 7 7 6 10/30 Write block 573 len 16384 data: 7 6 9 9 11/30 Write block 201 len 4096 data: 9 12/30 Write block 325 len 24576 data: 4 3 2 3 4 5 13/30 Write block 322 len 28672 data: 9 7 4 5 4 3 4 14/30 Write block 502 len 32768 data: 11 14 12 12 10 6 5 4 15/30 Read block 264 len 16384 16/30 Read block 579 len 12288 17/30 Write block 442 len 16384 data: 5 5 8 9 18/30 Read block 409 len 36864 19/30 Write block 144 len 32768 data: 3 4 4 4 6 5 5 5 20/30 Write block 574 len 24576 data: 7 10 10 7 8 7 21/30 Read block 152 len 28672 22/30 Write block 75 len 20480 data: 5 5 8 8 6 23/30 Read block 293 len 32768 24/30 Read block 273 len 4096 25/30 Write block 584 len 8192 data: 7 7 26/30 Write block 427 len 20480 data: 9 7 8 7 9 27/30 Read block 326 len 40960 28/30 Read block 232 len 40960 29/30 Read block 147 len 36864 30/30 Read block 475 len 28672 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 10 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:52.274058254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1510} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 137 137 137 F F F 1 020-039 75 75 75 137 137 137 F F F 2 040-059 75 75 75 137 137 137 T T F <--- 3 060-079 75 75 75 137 137 137 T T F <--- 4 080-099 75 75 75 137 137 137 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 69 69 69 123 123 123 T T F <--- 8 160-179 76 76 74 139 139 136 F F F <--- 9 180-199 70 70 70 124 124 124 F F F 10 200-219 76 76 75 138 138 137 T T F <--- 11 220-239 66 66 66 119 119 119 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 76 76 75 138 138 137 T T T <--- 17 340-359 75 75 75 137 137 137 F F F 18 360-379 73 73 73 135 135 135 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 75 75 75 137 137 137 T T F <--- 22 440-459 75 75 75 137 137 137 T T F <--- 23 460-479 75 75 75 137 137 137 F F F 24 480-499 72 72 72 130 130 130 F F F 25 500-519 68 68 68 121 121 121 T T F <--- 26 520-539 73 73 73 135 135 135 F F F 27 540-559 71 71 71 129 129 129 F F F 28 560-579 72 72 72 133 133 133 T T F <--- 29 580-599 75 75 75 137 137 137 T T F <--- Max gen: 76, Max flush: 139 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 137 137 137 F F F 1 020-039 75 75 75 137 137 137 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 77 77 77 140 140 140 F F F 4 080-099 75 75 75 137 137 137 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 77 77 77 140 140 140 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 77 77 77 140 140 140 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 75 75 75 137 137 137 F F F 16 320-339 77 77 77 140 140 140 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 73 73 73 135 135 135 F F F 19 380-399 66 66 66 119 119 119 F F F 20 400-419 66 66 66 119 119 119 F F F 21 420-439 77 77 77 140 140 140 F F F 22 440-459 77 77 77 140 140 140 F F F 23 460-479 75 75 75 137 137 137 F F F 24 480-499 72 72 72 130 130 130 F F F 25 500-519 77 77 77 140 140 140 F F F 26 520-539 73 73 73 135 135 135 F F F 27 540-559 71 71 71 129 129 129 F F F 28 560-579 77 77 77 140 140 140 F F F 29 580-599 77 77 77 140 140 140 F F F Max gen: 77, Max flush: 140 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.703300681Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.750417376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.751526774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.751586162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.751606121Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.75163387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} {"msg":"Crucible e5cf9f57-1abc-40fc-abd7-e956e11ed4da has session id: caf3e63f-a895-4a30-a8ee-a7753f369a4a","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752274455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Upstairs opts: Upstairs UUID: e5cf9f57-1abc-40fc-abd7-e956e11ed4da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752324613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Crucible stats registered with UUID: e5cf9f57-1abc-40fc-abd7-e956e11ed4da","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752349123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:56.752365382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752571984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752614393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752634542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752814655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752835464Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} {"msg":"e5cf9f57-1abc-40fc-abd7-e956e11ed4da active request set","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752912431Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.75293288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"0","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.75295318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.752981579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"2","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.753012397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.753043926Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.753071705Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64905)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.753173941Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33758)","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.75321883Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:56.753259728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:56.753294136Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-14T23:59:56.753326045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.753349734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.753388773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.754323757Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"0","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.754436633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"2","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.754920655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-14T23:59:56.755005351Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.755212073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39006)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.755440495Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.756905179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 140, 140, 137, 137, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757625061Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[0]R generation[0..12]: [75, 75, 77, 77, 75, 75, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.7576693Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757704778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 140, 140, 137, 137, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757730328Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[1]R generation[0..12]: [75, 75, 77, 77, 75, 75, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757753607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757778246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 140, 140, 137, 137, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757801835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[2]R generation[0..12]: [75, 75, 77, 77, 75, 75, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757824584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757848983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757873692Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757894701Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.75791473Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.75793512Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"":"downstairs","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757956119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757975018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"0","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.757995987Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"1","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758016147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"client":"2","":"downstairs","session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758035846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758052615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 e5cf9f57-1abc-40fc-abd7-e956e11ed4da is now active with session: caf3e63f-a895-4a30-a8ee-a7753f369a4a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758071515Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758087164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518,"session_id":"caf3e63f-a895-4a30-a8ee-a7753f369a4a"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758103283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} {"msg":"Activated sub_volume e5cf9f57-1abc-40fc-abd7-e956e11ed4da","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:06.758117043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 8192 data: 9 10 02/30 Write block 571 len 28672 data: 9 7 8 8 11 11 8 03/30 Write block 548 len 20480 data: 4 5 6 7 7 04/30 Write block 515 len 32768 data: 7 7 7 8 5 5 3 4 05/30 Write block 14 len 32768 data: 9 9 9 9 5 6 8 9 06/30 Read block 461 len 28672 07/30 Flush 08/30 Write block 319 len 12288 data: 9 9 10 09/30 Flush 10/30 Write block 593 len 4096 data: 8 11/30 Write block 370 len 16384 data: 4 6 7 7 12/30 Flush 13/30 Write block 34 len 4096 data: 3 14/30 Read block 111 len 40960 15/30 Read block 587 len 36864 16/30 Write block 85 len 24576 data: 7 7 6 6 6 4 17/30 Write block 364 len 8192 data: 6 6 18/30 Write block 574 len 16384 data: 9 12 12 9 19/30 Write block 410 len 12288 data: 6 7 5 20/30 Read block 232 len 36864 21/30 Write block 380 len 12288 data: 3 4 2 22/30 Write block 7 len 28672 data: 6 7 7 7 7 9 9 23/30 Flush 24/30 Write block 477 len 32768 data: 4 3 5 4 4 5 8 9 25/30 Read block 362 len 8192 26/30 Write block 350 len 24576 data: 8 7 8 7 6 4 27/30 Read block 391 len 16384 28/30 Write block 348 len 36864 data: 9 8 9 8 9 8 7 5 5 29/30 Read block 172 len 40960 30/30 Write block 66 len 16384 data: 7 7 5 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:06.776755463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1518} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 75 78 144 137 144 F F F <--- 1 020-039 78 75 78 144 137 144 F F F <--- 2 040-059 77 77 77 140 140 140 F F F 3 060-079 77 77 77 140 140 140 T F T <--- 4 080-099 78 75 78 144 137 144 F F F <--- 5 100-119 75 75 75 137 137 137 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 77 77 77 140 140 140 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 77 77 77 140 140 140 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 78 75 78 142 137 142 F F F <--- 16 320-339 78 77 78 142 140 142 F T F <--- 17 340-359 75 75 75 137 137 137 T F T <--- 18 360-379 78 73 78 144 135 144 F F F <--- 19 380-399 78 66 78 144 119 144 F F F <--- 20 400-419 78 66 78 144 119 144 F F F <--- 21 420-439 77 77 77 140 140 140 F F F 22 440-459 77 77 77 140 140 140 F F F 23 460-479 75 75 75 137 137 137 T F T <--- 24 480-499 72 72 72 130 130 130 T F T <--- 25 500-519 78 77 78 141 140 141 F F F <--- 26 520-539 78 73 78 141 135 141 F F F <--- 27 540-559 78 71 78 141 129 141 F F F <--- 28 560-579 78 77 78 144 140 144 F F F <--- 29 580-599 78 77 78 143 140 143 F F F <--- Max gen: 78, Max flush: 144 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 144 144 144 F F F 1 020-039 78 78 78 144 144 144 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 79 79 79 145 145 145 F F F 4 080-099 78 78 78 144 144 144 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 77 77 77 140 140 140 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 77 77 77 140 140 140 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 78 78 78 142 142 142 F F F 17 340-359 79 79 79 145 145 145 F F F 18 360-379 78 78 78 144 144 144 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 78 78 78 144 144 144 F F F 21 420-439 77 77 77 140 140 140 F F F 22 440-459 77 77 77 140 140 140 F F F 23 460-479 79 79 79 145 145 145 F F F 24 480-499 79 79 79 145 145 145 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 78 78 78 141 141 141 F F F 27 540-559 78 78 78 141 141 141 F F F 28 560-579 78 78 78 144 144 144 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 79, Max flush: 145 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.21517027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:11.258854076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.259240411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.29024703Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.290948253Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.290996332Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.291012271Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.29103424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} {"msg":"Crucible 2497b467-46a9-4d10-88fb-d2b9489ed04e has session id: e5bb5925-636a-47c9-b3b0-63350f1fbfb3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.291676916Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Upstairs opts: Upstairs UUID: 2497b467-46a9-4d10-88fb-d2b9489ed04e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.291740043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Crucible stats registered with UUID: 2497b467-46a9-4d10-88fb-d2b9489ed04e","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.291762873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:11.291779392Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.291958205Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292007983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292036762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292198856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292228595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} {"msg":"2497b467-46a9-4d10-88fb-d2b9489ed04e active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292286722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292307472Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"0","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292325221Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"1","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292342071Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"2","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.29235864Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292375929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292423127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60371)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292465756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40673)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292499814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37952)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.292532443Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.293783016Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"2","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.293869242Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"1","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.29392725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"0","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294389773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294493369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[0]R flush_numbers[0..12]: [144, 144, 140, 145, 144, 137, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294580915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[0]R generation[0..12]: [78, 78, 77, 79, 78, 75, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294607834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294627923Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[1]R flush_numbers[0..12]: [144, 144, 140, 145, 144, 137, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294647013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[1]R generation[0..12]: [78, 78, 77, 79, 78, 75, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294665332Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294684811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[2]R flush_numbers[0..12]: [144, 144, 140, 145, 144, 137, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.29471003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[2]R generation[0..12]: [78, 78, 77, 79, 78, 75, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294729149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294748849Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294772058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294802617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294832065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294850535Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"":"downstairs","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:00:11.294877674Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294911722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"0","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294939992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"1","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.294958301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"client":"2","":"downstairs","session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.29498381Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.295013779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"2497b467-46a9-4d10-88fb-d2b9489ed04e is now active with session: e5bb5925-636a-47c9-b3b0-63350f1fbfb3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.295039008Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.295054647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526,"session_id":"e5bb5925-636a-47c9-b3b0-63350f1fbfb3"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.295070647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} {"msg":"Activated sub_volume 2497b467-46a9-4d10-88fb-d2b9489ed04e","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:11.295088016Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 559 len 40960 data: 12 10 10 11 10 8 5 4 5 6 02/30 Read block 327 len 32768 03/30 Write block 245 len 28672 data: 4 6 5 5 5 5 6 04/30 Read block 296 len 8192 05/30 Read block 503 len 32768 06/30 Write block 495 len 20480 data: 7 8 8 7 5 07/30 Read block 125 len 8192 08/30 Read block 519 len 16384 09/30 Write block 38 len 8192 data: 5 5 10/30 Read block 343 len 24576 11/30 Write block 484 len 40960 data: 10 11 11 14 14 13 13 12 11 9 12/30 Write block 402 len 8192 data: 6 8 13/30 Read block 480 len 16384 14/30 Write block 424 len 20480 data: 10 7 7 10 8 15/30 Write block 28 len 36864 data: 8 8 7 5 6 4 4 3 3 16/30 Read block 164 len 12288 17/30 Flush 18/30 Write block 368 len 16384 data: 7 5 5 7 19/30 Read block 34 len 12288 20/30 Write block 78 len 16384 data: 9 7 6 5 21/30 Write block 408 len 20480 data: 7 6 7 8 6 22/30 Write block 490 len 28672 data: 14 13 12 10 8 8 9 23/30 Read block 275 len 36864 24/30 Read block 389 len 4096 25/30 Write block 47 len 40960 data: 9 10 10 9 10 10 10 11 8 5 26/30 Write block 456 len 8192 data: 6 4 27/30 Write block 72 len 12288 data: 8 9 7 28/30 Write block 99 len 24576 data: 6 8 7 7 7 6 29/30 Read block 115 len 40960 30/30 Read block 526 len 32768 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:11.314645541Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1526} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 144 144 144 F F F 1 020-039 78 80 80 144 146 146 F F F <--- 2 040-059 77 77 77 140 140 140 F T T <--- 3 060-079 79 79 79 145 145 145 F T T <--- 4 080-099 78 78 78 144 144 144 F T T <--- 5 100-119 75 75 75 137 137 137 F T T <--- 6 120-139 75 75 75 137 137 137 F F F 7 140-159 77 77 77 140 140 140 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 77 77 77 140 140 140 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 75 80 80 137 146 146 F F F <--- 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 78 78 78 142 142 142 F F F 17 340-359 79 79 79 145 145 145 F F F 18 360-379 78 78 78 144 144 144 F T T <--- 19 380-399 78 78 78 144 144 144 F F F 20 400-419 78 80 80 144 146 146 F T T <--- 21 420-439 77 80 80 140 146 146 F F F <--- 22 440-459 77 77 77 140 140 140 F T T <--- 23 460-479 79 79 79 145 145 145 F F F 24 480-499 79 80 80 145 146 146 F T T <--- 25 500-519 78 78 78 141 141 141 F F F 26 520-539 78 78 78 141 141 141 F F F 27 540-559 78 80 80 141 146 146 T F F <--- 28 560-579 78 80 80 144 146 146 T F F <--- 29 580-599 78 78 78 143 143 143 F F F Max gen: 80, Max flush: 146 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 144 144 144 F F F 1 020-039 80 80 80 146 146 146 F F F 2 040-059 81 81 81 147 147 147 F F F 3 060-079 81 81 81 147 147 147 F F F 4 080-099 81 81 81 147 147 147 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 77 77 77 140 140 140 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 77 77 77 140 140 140 F F F 11 220-239 66 66 66 119 119 119 F F F 12 240-259 80 80 80 146 146 146 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 73 73 73 135 135 135 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 78 78 78 142 142 142 F F F 17 340-359 79 79 79 145 145 145 F F F 18 360-379 81 81 81 147 147 147 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 81 81 81 147 147 147 F F F 21 420-439 80 80 80 146 146 146 F F F 22 440-459 81 81 81 147 147 147 F F F 23 460-479 79 79 79 145 145 145 F F F 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 78 78 78 141 141 141 F F F 27 540-559 80 80 80 146 146 146 F F F 28 560-579 80 80 80 146 146 146 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 81, Max flush: 147 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.775194155Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.820235479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821084086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821133735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821150284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821173463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} {"msg":"Crucible 5ebfc97b-ae94-4e99-812f-bcf63cf42576 has session id: 4a144455-381e-4340-ac03-26e7d2b4cb61","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821666364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Upstairs opts: Upstairs UUID: 5ebfc97b-ae94-4e99-812f-bcf63cf42576, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821731572Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Crucible stats registered with UUID: 5ebfc97b-ae94-4e99-812f-bcf63cf42576","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821768451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:15.82178869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821947044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.821993292Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822013771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822174615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822195214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} {"msg":"5ebfc97b-ae94-4e99-812f-bcf63cf42576 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822270881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822291911Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"0","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.82230989Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"1","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822335339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"2","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822369897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822401076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822432235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"ds_connection connected from Ok(127.0.0.1:37732)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822510682Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"ds_connection connected from Ok(127.0.0.1:44673)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.822549441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"ds_connection connected from Ok(127.0.0.1:36261)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.82257944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.82389054Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"1","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.823938598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"0","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.824488127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.824525806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.825875554Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"2","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[0]R flush_numbers[0..12]: [144, 146, 147, 147, 147, 147, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826415894Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[0]R generation[0..12]: [78, 80, 81, 81, 81, 81, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826445132Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826466652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[1]R flush_numbers[0..12]: [144, 146, 147, 147, 147, 147, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826485631Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[1]R generation[0..12]: [78, 80, 81, 81, 81, 81, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.82650388Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826523349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[2]R flush_numbers[0..12]: [144, 146, 147, 147, 147, 147, 137, 140, 139, 124, 140, 119]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826542409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[2]R generation[0..12]: [78, 80, 81, 81, 81, 81, 75, 77, 76, 70, 77, 66]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826560538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826579897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826596336Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826612976Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Next flush: 148","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-02-15T00:00:15.826635425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826654894Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"":"downstairs","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826671954Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826687083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"0","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826703992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"1","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826720862Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"client":"2","":"downstairs","session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826737761Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.82675381Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"5ebfc97b-ae94-4e99-812f-bcf63cf42576 is now active with session: 4a144455-381e-4340-ac03-26e7d2b4cb61","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.82677164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826786909Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534,"session_id":"4a144455-381e-4340-ac03-26e7d2b4cb61"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826806419Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} {"msg":"Activated sub_volume 5ebfc97b-ae94-4e99-812f-bcf63cf42576","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:15.826820628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 530 len 28672 data: 5 5 5 5 7 7 8 02/30 Flush 03/30 Flush 04/30 Write block 0 len 8192 data: 4 6 05/30 Read block 578 len 12288 06/30 Read block 160 len 36864 07/30 Read block 177 len 12288 08/30 Write block 234 len 4096 data: 6 09/30 Read block 576 len 32768 10/30 Write block 199 len 40960 data: 7 10 10 8 8 10 11 9 8 8 11/30 Write block 299 len 12288 data: 11 12 13 12/30 Write block 434 len 8192 data: 10 11 13/30 Write block 462 len 24576 data: 5 5 4 4 3 4 14/30 Read block 506 len 12288 15/30 Write block 362 len 8192 data: 6 7 16/30 Read block 425 len 24576 17/30 Read block 223 len 32768 18/30 Write block 553 len 40960 data: 7 8 7 7 9 11 13 11 11 12 19/30 Read block 264 len 28672 20/30 Read block 342 len 40960 21/30 Write block 355 len 20480 data: 6 6 5 6 6 22/30 Flush 23/30 Write block 148 len 12288 data: 7 6 6 24/30 Read block 245 len 28672 25/30 Flush 26/30 Write block 424 len 12288 data: 11 8 8 27/30 Write block 38 len 28672 data: 6 6 8 8 7 8 8 28/30 Write block 530 len 36864 data: 6 6 6 6 8 8 9 9 9 29/30 Read block 238 len 20480 30/30 Read block 388 len 40960 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:15.847044468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1534} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 78 150 150 144 F F F <--- 1 020-039 80 80 80 146 146 146 T T F <--- 2 040-059 81 81 81 147 147 147 T T F <--- 3 060-079 81 81 81 147 147 147 F F F 4 080-099 81 81 81 147 147 147 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 82 82 77 151 151 140 F F F <--- 8 160-179 76 76 76 139 139 139 F F F 9 180-199 82 82 70 150 150 124 F F F <--- 10 200-219 82 82 77 150 150 140 F F F <--- 11 220-239 82 82 66 150 150 119 F F F <--- 12 240-259 80 80 80 146 146 146 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 82 82 73 150 150 135 F F F <--- 15 300-319 82 82 78 150 150 142 F F F <--- 16 320-339 78 78 78 142 142 142 F F F 17 340-359 82 82 79 150 150 145 F F F <--- 18 360-379 82 82 81 150 150 147 F F F <--- 19 380-399 78 78 78 144 144 144 F F F 20 400-419 81 81 81 147 147 147 F F F 21 420-439 82 82 80 150 150 146 T T F <--- 22 440-459 81 81 81 147 147 147 F F F 23 460-479 82 82 79 150 150 145 F F F <--- 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 82 82 78 148 148 141 T T T <--- 27 540-559 82 82 80 150 150 146 F F F <--- 28 560-579 82 82 80 150 150 146 F F F <--- 29 580-599 78 78 78 143 143 143 F F F Max gen: 82, Max flush: 151 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 F F F 1 020-039 83 83 83 152 152 152 F F F 2 040-059 83 83 83 152 152 152 F F F 3 060-079 81 81 81 147 147 147 F F F 4 080-099 81 81 81 147 147 147 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 82 82 82 151 151 151 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 82 82 82 150 150 150 F F F 12 240-259 80 80 80 146 146 146 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 82 82 82 150 150 150 F F F 15 300-319 82 82 82 150 150 150 F F F 16 320-339 78 78 78 142 142 142 F F F 17 340-359 82 82 82 150 150 150 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 81 81 81 147 147 147 F F F 21 420-439 83 83 83 152 152 152 F F F 22 440-459 81 81 81 147 147 147 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 83 83 83 152 152 152 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 82 82 82 150 150 150 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 83, Max flush: 152 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.292865374Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.339532266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.340606735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.340669553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.340686572Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.340709571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} {"msg":"Crucible 3a414b37-0ddb-479e-b6a1-327909cb3e2e has session id: b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.3412639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Upstairs opts: Upstairs UUID: 3a414b37-0ddb-479e-b6a1-327909cb3e2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341322928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Crucible stats registered with UUID: 3a414b37-0ddb-479e-b6a1-327909cb3e2e","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341346897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:20.341363306Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.34152888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341581348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341610547Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341732472Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341752321Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} {"msg":"3a414b37-0ddb-479e-b6a1-327909cb3e2e active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341821009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341840248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"0","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341860747Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"1","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341889266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341921325Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341952784Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.341990702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:20.342070459Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:20.342120917Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45115)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.342153876Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:20.342189815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59144)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.342218794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.34230848Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.342373397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.343397529Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"1","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.343485415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"0","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.344061223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:20.344091672Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.342811531Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46228)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.342926066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.344778476Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[0]R flush_numbers[0..12]: [150, 152, 152, 147, 147, 147, 137, 151, 139, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345518548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[0]R generation[0..12]: [82, 83, 83, 81, 81, 81, 75, 82, 76, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345563576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345590345Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[1]R flush_numbers[0..12]: [150, 152, 152, 147, 147, 147, 137, 151, 139, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345614444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[1]R generation[0..12]: [82, 83, 83, 81, 81, 81, 75, 82, 76, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345637643Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345662102Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[2]R flush_numbers[0..12]: [150, 152, 152, 147, 147, 147, 137, 151, 139, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345685741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[2]R generation[0..12]: [82, 83, 83, 81, 81, 81, 75, 82, 76, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.34570888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.34573361Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345770428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345792907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:00:30.345813227Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345839095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"":"downstairs","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345866014Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345885554Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"0","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345907203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"1","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345928632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"client":"2","":"downstairs","session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.345950141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.34597019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"3a414b37-0ddb-479e-b6a1-327909cb3e2e is now active with session: b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.34599243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.346011619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542,"session_id":"b34ae4d0-cb72-43a6-a6e5-1ed8f25b9a2a"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.346031498Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} {"msg":"Activated sub_volume 3a414b37-0ddb-479e-b6a1-327909cb3e2e","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:30.346049587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 24576 data: 5 3 4 7 5 4 02/30 Write block 416 len 28672 data: 5 6 6 7 7 8 9 03/30 Write block 21 len 36864 data: 10 8 7 9 9 9 8 9 9 04/30 Read block 137 len 28672 05/30 Read block 13 len 32768 06/30 Read block 500 len 32768 07/30 Read block 310 len 12288 08/30 Flush 09/30 Write block 91 len 24576 data: 4 5 5 6 8 5 10/30 Read block 481 len 40960 11/30 Flush 12/30 Write block 16 len 24576 data: 10 10 6 7 9 11 13/30 Write block 250 len 12288 data: 6 7 7 14/30 Read block 218 len 32768 15/30 Read block 544 len 36864 16/30 Write block 448 len 16384 data: 7 6 5 6 17/30 Read block 182 len 20480 18/30 Flush 19/30 Write block 336 len 40960 data: 7 9 8 7 9 7 8 8 8 8 20/30 Flush 21/30 Write block 568 len 40960 data: 7 8 9 10 8 9 10 13 13 10 22/30 Read block 332 len 36864 23/30 Write block 522 len 4096 data: 5 24/30 Flush 25/30 Write block 272 len 24576 data: 8 7 7 7 5 6 26/30 Read block 91 len 8192 27/30 Read block 195 len 16384 28/30 Read block 537 len 12288 29/30 Read block 206 len 20480 30/30 Write block 222 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 9 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 5 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 28 31 Done 29 28 2 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:30.366558196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1542} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 82 155 155 150 F F F <--- 1 020-039 84 84 83 155 155 152 F F F <--- 2 040-059 83 83 83 152 152 152 F F F 3 060-079 81 81 81 147 147 147 F F F 4 080-099 84 84 81 154 154 147 F F F <--- 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 82 82 82 151 151 151 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 82 82 82 150 150 150 T T F <--- 12 240-259 84 84 80 155 155 146 F F F <--- 13 260-279 68 68 68 121 121 121 T T F <--- 14 280-299 82 82 82 150 150 150 F F F 15 300-319 82 82 82 150 150 150 F F F 16 320-339 84 84 78 156 156 142 F F F <--- 17 340-359 84 84 82 156 156 150 F F F <--- 18 360-379 82 82 82 150 150 150 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 84 84 81 153 153 147 F F T <--- 21 420-439 84 84 83 153 153 152 F F T <--- 22 440-459 84 84 81 155 155 147 F F T <--- 23 460-479 82 82 82 150 150 150 F F F 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 84 84 83 157 157 152 F F F <--- 27 540-559 82 82 82 150 150 150 F F F 28 560-579 84 84 82 157 157 150 F F F <--- 29 580-599 78 78 78 143 143 143 F F F Max gen: 84, Max flush: 157 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 155 155 155 F F F 1 020-039 84 84 84 155 155 155 F F F 2 040-059 83 83 83 152 152 152 F F F 3 060-079 81 81 81 147 147 147 F F F 4 080-099 84 84 84 154 154 154 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 82 82 82 151 151 151 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 85 85 85 158 158 158 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 85 85 85 158 158 158 F F F 14 280-299 82 82 82 150 150 150 F F F 15 300-319 82 82 82 150 150 150 F F F 16 320-339 84 84 84 156 156 156 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 84 84 84 153 153 153 F F F 21 420-439 84 84 84 153 153 153 F F F 22 440-459 84 84 84 155 155 155 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 84 84 84 157 157 157 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 84 84 84 157 157 157 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 85, Max flush: 158 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.782038697Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.827468467Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.828283896Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.828352143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.828375682Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.828404501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} {"msg":"Crucible a8d9549f-6628-4e0d-82dc-1ae7dad0dedf has session id: 5737afe1-9582-4d9f-93d2-3013e10805bd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.828936681Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Upstairs opts: Upstairs UUID: a8d9549f-6628-4e0d-82dc-1ae7dad0dedf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829004518Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Crucible stats registered with UUID: a8d9549f-6628-4e0d-82dc-1ae7dad0dedf","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829041867Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:34.829075995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829247209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829298807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829320376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.82948844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829515618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} {"msg":"a8d9549f-6628-4e0d-82dc-1ae7dad0dedf active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829569376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829599645Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"0","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829618535Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829636174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"2","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829652613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829673623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829706091Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35810)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829860596Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35189)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.829987531Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:34.830032389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:34.830071697Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:34.830109716Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.830191963Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.830239631Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.831208264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"0","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.831258092Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"2","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.831795732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:34.831887148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.830964623Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55894)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.831070169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.83288907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[0]R flush_numbers[0..12]: [155, 155, 152, 147, 154, 147, 137, 151, 139, 150, 150, 158]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833579294Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[0]R generation[0..12]: [84, 84, 83, 81, 84, 81, 75, 82, 76, 82, 82, 85]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833625172Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833658121Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[1]R flush_numbers[0..12]: [155, 155, 152, 147, 154, 147, 137, 151, 139, 150, 150, 158]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.83368934Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[1]R generation[0..12]: [84, 84, 83, 81, 84, 81, 75, 82, 76, 82, 82, 85]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833720278Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833753037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[2]R flush_numbers[0..12]: [155, 155, 152, 147, 154, 147, 137, 151, 139, 150, 150, 158]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833783246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[2]R generation[0..12]: [84, 84, 83, 81, 84, 81, 75, 82, 76, 82, 82, 85]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833812815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833848613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833882552Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-02-15T00:00:44.833915371Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833953579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.833985808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"":"downstairs","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834025607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834057345Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"0","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834091814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"1","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834123503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"client":"2","":"downstairs","session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834157431Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.83418946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"a8d9549f-6628-4e0d-82dc-1ae7dad0dedf is now active with session: 5737afe1-9582-4d9f-93d2-3013e10805bd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834222259Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834239899Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550,"session_id":"5737afe1-9582-4d9f-93d2-3013e10805bd"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834261108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} {"msg":"Activated sub_volume a8d9549f-6628-4e0d-82dc-1ae7dad0dedf","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:44.834275707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 302 len 32768 data: 13 10 10 11 9 11 10 12 02/30 Write block 178 len 8192 data: 5 5 03/30 Write block 89 len 4096 data: 7 04/30 Write block 228 len 36864 data: 5 5 8 9 7 5 7 8 8 05/30 Read block 406 len 12288 06/30 Write block 53 len 8192 data: 11 12 07/30 Flush 08/30 Write block 85 len 24576 data: 8 8 7 7 8 5 09/30 Read block 133 len 20480 10/30 Write block 306 len 32768 data: 10 12 11 13 11 12 13 13 11/30 Read block 96 len 12288 12/30 Read block 31 len 4096 13/30 Read block 163 len 24576 14/30 Read block 138 len 8192 15/30 Write block 18 len 16384 data: 7 8 10 12 16/30 Read block 202 len 40960 17/30 Write block 333 len 4096 data: 5 18/30 Flush 19/30 Flush 20/30 Write block 573 len 40960 data: 10 11 14 14 11 9 8 8 8 7 21/30 Write block 299 len 36864 data: 12 13 14 14 11 11 12 11 13 22/30 Read block 152 len 32768 23/30 Read block 86 len 40960 24/30 Read block 445 len 20480 25/30 Read block 120 len 12288 26/30 Write block 155 len 4096 data: 5 27/30 Read block 359 len 16384 28/30 Write block 4 len 8192 data: 5 6 29/30 Write block 445 len 40960 data: 10 10 9 8 7 6 7 5 6 4 30/30 Write block 155 len 24576 data: 6 5 4 5 8 8 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 23 2 27 Done 21 0 21 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1006)) Some(JobId(1018)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:44.854701469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1550} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 84 86 160 155 160 T F T <--- 1 020-039 86 84 86 160 155 160 F F F <--- 2 040-059 86 86 86 159 159 159 F F F 3 060-079 81 81 81 147 147 147 F F F 4 080-099 86 86 86 160 159 160 F F F <--- 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 82 82 82 151 151 151 T F T <--- 8 160-179 86 86 86 159 159 159 T F T <--- 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 86 86 86 159 159 159 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 85 85 85 158 158 158 F F F 14 280-299 82 82 82 150 150 150 T F T <--- 15 300-319 86 86 86 160 159 160 T F T <--- 16 320-339 86 84 86 160 156 160 F F F <--- 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 84 84 84 153 153 153 F F F 21 420-439 84 84 84 153 153 153 F F F 22 440-459 84 84 84 155 155 155 T F T <--- 23 460-479 82 82 82 150 150 150 F F F 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 84 84 84 157 157 157 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 84 84 84 157 157 157 T F T <--- 29 580-599 78 78 78 143 143 143 T F T <--- Max gen: 86, Max flush: 160 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 86 86 86 160 160 160 F F F 2 040-059 86 86 86 159 159 159 F F F 3 060-079 81 81 81 147 147 147 F F F 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 87 87 87 161 161 161 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 86 86 86 159 159 159 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 85 85 85 158 158 158 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 87 87 87 161 161 161 F F F 16 320-339 86 86 86 160 160 160 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 78 78 78 144 144 144 F F F 20 400-419 84 84 84 153 153 153 F F F 21 420-439 84 84 84 153 153 153 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 81 81 81 147 147 147 F F F 25 500-519 78 78 78 141 141 141 F F F 26 520-539 84 84 84 157 157 157 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 87 87 87 161 161 161 F F F 29 580-599 87 87 87 161 161 161 F F F Max gen: 87, Max flush: 161 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.299933177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.345519351Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346295211Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346356869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346377708Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346406257Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} {"msg":"Crucible 9d807bfd-b24a-4b99-aab6-5988d6840e88 has session id: 215919a9-abb5-40d8-bd03-0ccd10811823","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346873979Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Upstairs opts: Upstairs UUID: 9d807bfd-b24a-4b99-aab6-5988d6840e88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346926977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Crucible stats registered with UUID: 9d807bfd-b24a-4b99-aab6-5988d6840e88","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.346947646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:49.346964336Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347203287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347255485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347289313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347551243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347585082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} {"msg":"9d807bfd-b24a-4b99-aab6-5988d6840e88 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347648859Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347682608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"0","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347715307Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"1","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347750886Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"2","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347783545Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347818493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347852852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"ds_connection connected from Ok(127.0.0.1:61208)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347886521Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"ds_connection connected from Ok(127.0.0.1:44496)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347922459Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"ds_connection connected from Ok(127.0.0.1:42875)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.347966138Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.349237339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"2","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.349279808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"0","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.349970931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.35000278Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"1","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350028219Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[0]R flush_numbers[0..12]: [161, 160, 159, 147, 160, 147, 137, 161, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350561919Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[0]R generation[0..12]: [87, 86, 86, 81, 86, 81, 75, 87, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350591787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350612117Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[1]R flush_numbers[0..12]: [161, 160, 159, 147, 160, 147, 137, 161, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350631056Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[1]R generation[0..12]: [87, 86, 86, 81, 86, 81, 75, 87, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350649255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350668924Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[2]R flush_numbers[0..12]: [161, 160, 159, 147, 160, 147, 137, 161, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350687624Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[2]R generation[0..12]: [87, 86, 86, 81, 86, 81, 75, 87, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350710023Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350729842Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350745912Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350771341Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.35078752Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350803499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"":"downstairs","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350820389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350835408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"0","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350852228Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"1","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350869877Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"client":"2","":"downstairs","session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350886756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350902676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"9d807bfd-b24a-4b99-aab6-5988d6840e88 is now active with session: 215919a9-abb5-40d8-bd03-0ccd10811823","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350920475Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350935564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558,"session_id":"215919a9-abb5-40d8-bd03-0ccd10811823"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350951544Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} {"msg":"Activated sub_volume 9d807bfd-b24a-4b99-aab6-5988d6840e88","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:49.350965513Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 8192 data: 7 5 02/30 Read block 310 len 36864 03/30 Write block 392 len 16384 data: 5 7 7 8 04/30 Write block 483 len 16384 data: 9 11 12 12 05/30 Write block 60 len 4096 data: 11 06/30 Read block 498 len 40960 07/30 Read block 180 len 24576 08/30 Read block 468 len 12288 09/30 Read block 285 len 28672 10/30 Read block 134 len 4096 11/30 Read block 330 len 16384 12/30 Write block 53 len 32768 data: 12 13 9 6 5 8 9 12 13/30 Write block 391 len 32768 data: 6 6 8 8 9 10 9 8 14/30 Read block 453 len 24576 15/30 Write block 525 len 8192 data: 6 5 16/30 Write block 75 len 8192 data: 6 6 17/30 Read block 57 len 16384 18/30 Read block 119 len 28672 19/30 Read block 536 len 16384 20/30 Read block 465 len 4096 21/30 Read block 334 len 8192 22/30 Read block 28 len 36864 23/30 Read block 32 len 24576 24/30 Read block 564 len 20480 25/30 Write block 274 len 24576 data: 8 8 6 7 5 4 26/30 Flush 27/30 Write block 516 len 32768 data: 8 8 9 6 6 4 6 4 28/30 Write block 394 len 32768 data: 9 10 11 10 9 8 7 5 29/30 Write block 142 len 24576 data: 6 4 4 5 5 5 30/30 Read block 571 len 28672 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 2 32 Done 30 0 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:49.369082323Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1558} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 86 86 86 160 160 160 F F F 2 040-059 88 86 88 162 159 162 F F F <--- 3 060-079 88 81 88 162 147 162 F F F <--- 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 87 87 87 161 161 161 T F T <--- 8 160-179 87 87 87 161 161 161 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 86 86 86 159 159 159 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 88 85 88 162 158 162 F F F <--- 14 280-299 87 87 87 161 161 161 F F F 15 300-319 87 87 87 161 161 161 F F F 16 320-339 86 86 86 160 160 160 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 88 78 88 162 144 162 T F T <--- 20 400-419 84 84 84 153 153 153 T F T <--- 21 420-439 84 84 84 153 153 153 F F F 22 440-459 88 87 88 162 161 162 F F F <--- 23 460-479 82 82 82 150 150 150 F F F 24 480-499 88 81 88 162 147 162 F F F <--- 25 500-519 78 78 78 141 141 141 T F T <--- 26 520-539 88 84 88 162 157 162 T F T <--- 27 540-559 82 82 82 150 150 150 F F F 28 560-579 87 87 87 161 161 161 F F F 29 580-599 87 87 87 161 161 161 F F F Max gen: 88, Max flush: 162 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 86 86 86 160 160 160 F F F 2 040-059 88 88 88 162 162 162 F F F 3 060-079 88 88 88 162 162 162 F F F 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 89 89 89 163 163 163 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 86 86 86 159 159 159 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 88 88 88 162 162 162 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 87 87 87 161 161 161 F F F 16 320-339 86 86 86 160 160 160 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 89 89 89 163 163 163 F F F 20 400-419 89 89 89 163 163 163 F F F 21 420-439 84 84 84 153 153 153 F F F 22 440-459 88 88 88 162 162 162 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 88 88 88 162 162 162 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 89 89 89 163 163 163 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 87 87 87 161 161 161 F F F 29 580-599 87 87 87 161 161 161 F F F Max gen: 89, Max flush: 163 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.775449722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.824261182Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.825412728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.825466486Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.825483085Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.825505075Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} {"msg":"Crucible 8d599b49-fcbc-4808-859b-08c118426a86 has session id: 623a2c5f-d165-4ce3-aaa3-9f480dc653eb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826134831Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Upstairs opts: Upstairs UUID: 8d599b49-fcbc-4808-859b-08c118426a86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826202418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Crucible stats registered with UUID: 8d599b49-fcbc-4808-859b-08c118426a86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826228817Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:53.826245866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826451008Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826504746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826536805Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826705779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826740707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} {"msg":"8d599b49-fcbc-4808-859b-08c118426a86 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826784666Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826804825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"0","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826822164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"1","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826839144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"2","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826858453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.826888402Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.82691861Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57344)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.827126703Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62402)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.827390783Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51033)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.827420951Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.828445263Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"0","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.828573487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"1","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.828603106Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"2","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829074728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829177695Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[0]R flush_numbers[0..12]: [161, 160, 162, 162, 160, 147, 137, 163, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829230073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[0]R generation[0..12]: [87, 86, 88, 88, 86, 81, 75, 89, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829252752Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829273181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[1]R flush_numbers[0..12]: [161, 160, 162, 162, 160, 147, 137, 163, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.82929229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[1]R generation[0..12]: [87, 86, 88, 88, 86, 81, 75, 89, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829310899Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829330559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[2]R flush_numbers[0..12]: [161, 160, 162, 162, 160, 147, 137, 163, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829349398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[2]R generation[0..12]: [87, 86, 88, 88, 86, 81, 75, 89, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829367707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829387247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829403366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829422925Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829450764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:00:53.829472823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"":"downstairs","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829497582Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829515942Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"0","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829533091Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"1","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.8295496Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"client":"2","":"downstairs","session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.82956688Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829583009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"8d599b49-fcbc-4808-859b-08c118426a86 is now active with session: 623a2c5f-d165-4ce3-aaa3-9f480dc653eb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829600848Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829615978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566,"session_id":"623a2c5f-d165-4ce3-aaa3-9f480dc653eb"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829631997Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} {"msg":"Activated sub_volume 8d599b49-fcbc-4808-859b-08c118426a86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:53.829645847Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 24576 data: 10 10 8 6 7 5 02/30 Write block 306 len 4096 data: 12 03/30 Read block 246 len 4096 04/30 Read block 395 len 8192 05/30 Read block 78 len 8192 06/30 Write block 136 len 36864 data: 8 7 8 7 6 6 7 5 5 07/30 Read block 144 len 12288 08/30 Flush 09/30 Write block 577 len 16384 data: 12 10 9 9 10/30 Write block 416 len 24576 data: 6 7 7 8 8 9 11/30 Read block 551 len 16384 12/30 Read block 177 len 8192 13/30 Read block 280 len 32768 14/30 Flush 15/30 Write block 570 len 12288 data: 10 11 9 16/30 Read block 40 len 24576 17/30 Read block 410 len 40960 18/30 Write block 22 len 28672 data: 9 8 10 10 10 9 11 19/30 Read block 140 len 16384 20/30 Read block 309 len 16384 21/30 Read block 509 len 36864 22/30 Write block 439 len 40960 data: 9 9 7 6 6 9 11 11 10 9 23/30 Flush 24/30 Write block 273 len 24576 data: 8 9 9 7 8 6 25/30 Flush 26/30 Write block 317 len 36864 data: 7 8 10 10 11 11 8 5 6 27/30 Write block 49 len 24576 data: 11 10 11 11 13 14 28/30 Read block 181 len 8192 29/30 Read block 318 len 24576 30/30 Read block 291 len 28672 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:53.851031642Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1566} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 90 86 90 166 160 166 F T F <--- 2 040-059 88 88 88 162 162 162 T F T <--- 3 060-079 88 88 88 162 162 162 F F F 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 90 75 90 164 137 164 F F F <--- 7 140-159 90 89 90 164 163 164 F F F <--- 8 160-179 87 87 87 161 161 161 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 86 86 86 159 159 159 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 90 88 90 167 162 167 F F F <--- 14 280-299 87 87 87 161 161 161 F F F 15 300-319 90 87 90 164 161 164 T T T <--- 16 320-339 86 86 86 160 160 160 T F T <--- 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 89 89 89 163 163 163 F F F 20 400-419 90 89 90 165 163 165 F F F <--- 21 420-439 90 84 90 166 153 166 F F F <--- 22 440-459 90 88 90 166 162 166 F F F <--- 23 460-479 82 82 82 150 150 150 F F F 24 480-499 88 88 88 162 162 162 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 89 89 89 163 163 163 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 90 87 90 166 161 166 F F F <--- 29 580-599 90 87 90 165 161 165 F F F <--- Max gen: 90, Max flush: 167 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 90 90 90 166 166 166 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 88 88 88 162 162 162 F F F 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 90 90 90 164 164 164 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 87 87 87 161 161 161 F F F 9 180-199 82 82 82 150 150 150 F F F 10 200-219 82 82 82 150 150 150 F F F 11 220-239 86 86 86 159 159 159 F F F 12 240-259 84 84 84 155 155 155 F F F 13 260-279 90 90 90 167 167 167 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 91 91 91 168 168 168 F F F 16 320-339 91 91 91 168 168 168 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 82 82 82 150 150 150 F F F 19 380-399 89 89 89 163 163 163 F F F 20 400-419 90 90 90 165 165 165 F F F 21 420-439 90 90 90 166 166 166 F F F 22 440-459 90 90 90 166 166 166 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 88 88 88 162 162 162 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 89 89 89 163 163 163 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 91, Max flush: 168 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.289736779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.334229284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.334948006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335036143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335059312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335090051Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} {"msg":"Crucible 43ae95b0-9f53-45e4-8e7c-83547e94a9ea has session id: ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335553583Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Upstairs opts: Upstairs UUID: 43ae95b0-9f53-45e4-8e7c-83547e94a9ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335614551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Crucible stats registered with UUID: 43ae95b0-9f53-45e4-8e7c-83547e94a9ea","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.33563718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:58.335658549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335848812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.33589981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.335922549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336104612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336145291Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} {"msg":"43ae95b0-9f53-45e4-8e7c-83547e94a9ea active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336205289Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336234497Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"0","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336252527Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"1","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336269756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"2","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336295765Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336324854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336351073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59109)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336466448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34570)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336511817Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62595)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.336610263Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.337790398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"0","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.337841816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"2","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.338374286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.33852502Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.338662145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"1","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[0]R flush_numbers[0..12]: [161, 166, 168, 162, 160, 147, 164, 164, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339177035Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[0]R generation[0..12]: [87, 90, 91, 88, 86, 81, 90, 90, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339203404Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339223794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[1]R flush_numbers[0..12]: [161, 166, 168, 162, 160, 147, 164, 164, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339242903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[1]R generation[0..12]: [87, 90, 91, 88, 86, 81, 90, 90, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339261412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339280591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[2]R flush_numbers[0..12]: [161, 166, 168, 162, 160, 147, 164, 164, 161, 150, 150, 159]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.33931379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[2]R generation[0..12]: [87, 90, 91, 88, 86, 81, 90, 90, 87, 82, 82, 86]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339334659Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339354129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339370318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339393377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339409567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339425546Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"":"downstairs","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339442555Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339457745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"0","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339474954Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"1","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339491673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"client":"2","":"downstairs","session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339508833Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339524812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"43ae95b0-9f53-45e4-8e7c-83547e94a9ea is now active with session: ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339542651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339557901Z","hostname":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574,"session_id":"ce246d2e-ae3a-49a0-ae61-8cbb031cf2e9"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.33957977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} {"msg":"Activated sub_volume 43ae95b0-9f53-45e4-8e7c-83547e94a9ea","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:00:58.339593789Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 4096 data: 8 02/30 Read block 481 len 36864 03/30 Write block 476 len 40960 data: 4 5 4 6 5 5 6 10 12 13 04/30 Read block 428 len 8192 05/30 Read block 120 len 20480 06/30 Write block 215 len 16384 data: 7 6 7 6 07/30 Read block 31 len 40960 08/30 Write block 263 len 40960 data: 5 5 5 5 5 6 8 7 8 9 09/30 Read block 294 len 4096 10/30 Write block 316 len 24576 data: 6 8 9 11 11 12 11/30 Read block 317 len 36864 12/30 Read block 504 len 40960 13/30 Read block 47 len 32768 14/30 Read block 144 len 32768 15/30 Flush 16/30 Write block 374 len 20480 data: 6 6 6 6 6 17/30 Write block 467 len 32768 data: 5 4 4 5 6 6 5 5 18/30 Write block 305 len 20480 data: 13 13 14 12 14 19/30 Write block 158 len 32768 data: 6 9 9 10 9 8 7 7 20/30 Write block 535 len 20480 data: 9 10 10 10 9 21/30 Write block 151 len 28672 data: 6 6 7 5 7 6 5 22/30 Read block 159 len 24576 23/30 Flush 24/30 Write block 408 len 24576 data: 8 7 8 9 7 6 25/30 Write block 232 len 36864 data: 8 6 8 9 9 9 7 7 5 26/30 Write block 127 len 20480 data: 8 10 7 8 7 27/30 Read block 165 len 8192 28/30 Write block 185 len 20480 data: 5 6 5 6 7 29/30 Write block 403 len 28672 data: 9 8 9 8 8 9 8 30/30 Write block 137 len 40960 data: 8 9 8 7 7 8 6 6 6 6 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:00:58.358496799Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1574} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 90 90 90 166 166 166 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 88 88 88 162 162 162 F F F 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 90 90 90 164 164 164 T F T <--- 7 140-159 92 90 92 170 164 170 T F T <--- 8 160-179 92 87 92 170 161 170 F F F <--- 9 180-199 82 82 82 150 150 150 T F T <--- 10 200-219 92 82 92 169 150 169 F F F <--- 11 220-239 86 86 86 159 159 159 T F T <--- 12 240-259 84 84 84 155 155 155 T F T <--- 13 260-279 92 90 92 169 167 169 F F F <--- 14 280-299 87 87 87 161 161 161 F F F 15 300-319 92 91 92 170 168 170 F F F <--- 16 320-339 92 91 92 169 168 169 F F F <--- 17 340-359 84 84 84 156 156 156 F F F 18 360-379 92 82 92 170 150 170 F F F <--- 19 380-399 89 89 89 163 163 163 F F F 20 400-419 90 90 90 165 165 165 T F T <--- 21 420-439 90 90 90 166 166 166 F F F 22 440-459 90 90 90 166 166 166 F F F 23 460-479 92 82 92 170 150 170 F F F <--- 24 480-499 92 88 92 169 162 169 F F F <--- 25 500-519 89 89 89 163 163 163 F F F 26 520-539 92 89 92 170 163 170 F F F <--- 27 540-559 82 82 82 150 150 150 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 92, Max flush: 170 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 F F F 1 020-039 90 90 90 166 166 166 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 88 88 88 162 162 162 F F F 4 080-099 86 86 86 160 160 160 F F F 5 100-119 81 81 81 147 147 147 F F F 6 120-139 93 93 93 171 171 171 F F F 7 140-159 93 93 93 171 171 171 F F F 8 160-179 92 92 92 170 170 170 F F F 9 180-199 93 93 93 171 171 171 F F F 10 200-219 92 92 92 169 169 169 F F F 11 220-239 93 93 93 171 171 171 F F F 12 240-259 93 93 93 171 171 171 F F F 13 260-279 92 92 92 169 169 169 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 92 92 92 170 170 170 F F F 16 320-339 92 92 92 169 169 169 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 92 92 92 170 170 170 F F F 19 380-399 89 89 89 163 163 163 F F F 20 400-419 93 93 93 171 171 171 F F F 21 420-439 90 90 90 166 166 166 F F F 22 440-459 90 90 90 166 166 166 F F F 23 460-479 92 92 92 170 170 170 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 92 92 92 170 170 170 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 93, Max flush: 171 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.789532958Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.833731284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.834396519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.834467196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.834492115Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.834524494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} {"msg":"Crucible ea2c4ebe-fa69-4f3d-922e-18adbaf41d18 has session id: cf31f9bb-c887-4beb-a5d6-7653451e306c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.8351463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Upstairs opts: Upstairs UUID: ea2c4ebe-fa69-4f3d-922e-18adbaf41d18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835194718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Crucible stats registered with UUID: ea2c4ebe-fa69-4f3d-922e-18adbaf41d18","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835218687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:02.835238527Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835492207Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835558815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835602543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835769396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835797125Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} {"msg":"ea2c4ebe-fa69-4f3d-922e-18adbaf41d18 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835853083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835883622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"0","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835902431Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835919641Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"2","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.83593642Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835965479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.835997598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50311)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.836031066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49109)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.836064165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:02.836157902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:02.83620615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:02.836333005Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.836425031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.83645848Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.837174593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"0","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.83725687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"2","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.837763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:02.837842268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.838504082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64006)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.838631727Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840194438Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[0]R flush_numbers[0..12]: [161, 166, 168, 162, 160, 147, 171, 171, 170, 171, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840824404Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[0]R generation[0..12]: [87, 90, 91, 88, 86, 81, 93, 93, 92, 93, 92, 93]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840861612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840887611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[1]R flush_numbers[0..12]: [161, 166, 168, 162, 160, 147, 171, 171, 170, 171, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.84091121Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[1]R generation[0..12]: [87, 90, 91, 88, 86, 81, 93, 93, 92, 93, 92, 93]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840934349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840958599Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[2]R flush_numbers[0..12]: [161, 166, 168, 162, 160, 147, 171, 171, 170, 171, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.840987767Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[2]R generation[0..12]: [87, 90, 91, 88, 86, 81, 93, 93, 92, 93, 92, 93]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841011097Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:01:12.841035796Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841061215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841082054Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841102023Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841122002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"":"downstairs","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841142941Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841160781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"0","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.84117762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"1","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.84119417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"client":"2","":"downstairs","session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841211389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841227488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"ea2c4ebe-fa69-4f3d-922e-18adbaf41d18 is now active with session: cf31f9bb-c887-4beb-a5d6-7653451e306c","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-15T00:01:12.841245248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841264387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582,"session_id":"cf31f9bb-c887-4beb-a5d6-7653451e306c"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841282566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} {"msg":"Activated sub_volume ea2c4ebe-fa69-4f3d-922e-18adbaf41d18","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:12.841296426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 24576 data: 7 6 8 7 7 7 02/30 Write block 194 len 36864 data: 3 5 5 7 7 8 11 11 9 03/30 Write block 431 len 36864 data: 10 8 7 11 12 10 9 9 10 04/30 Read block 62 len 16384 05/30 Read block 317 len 16384 06/30 Read block 143 len 12288 07/30 Read block 296 len 12288 08/30 Read block 574 len 32768 09/30 Write block 184 len 12288 data: 4 6 7 10/30 Write block 113 len 20480 data: 5 5 5 7 10 11/30 Flush 12/30 Write block 17 len 32768 data: 11 8 9 11 13 10 9 11 13/30 Read block 129 len 40960 14/30 Read block 463 len 32768 15/30 Read block 19 len 12288 16/30 Read block 465 len 36864 17/30 Read block 513 len 36864 18/30 Read block 550 len 28672 19/30 Read block 568 len 28672 20/30 Write block 381 len 20480 data: 5 3 2 3 4 21/30 Read block 280 len 4096 22/30 Read block 135 len 24576 23/30 Read block 282 len 24576 24/30 Read block 532 len 4096 25/30 Write block 365 len 36864 data: 7 6 6 8 6 6 8 8 8 26/30 Write block 228 len 8192 data: 6 6 27/30 Read block 467 len 24576 28/30 Write block 462 len 16384 data: 6 6 5 5 29/30 Write block 73 len 40960 data: 10 8 7 7 9 10 8 7 6 7 30/30 Write block 525 len 40960 data: 7 6 4 4 7 7 7 7 7 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 25 2 29 Done 28 5 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) None Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:12.860898119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1582} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 161 161 161 T F T <--- 1 020-039 90 90 90 166 166 166 T F T <--- 2 040-059 91 91 91 168 168 168 F F F 3 060-079 88 88 88 162 162 162 T F T <--- 4 080-099 86 86 86 160 160 160 T F T <--- 5 100-119 94 81 94 172 147 172 F F F <--- 6 120-139 93 93 93 171 171 171 F F F 7 140-159 94 93 94 172 171 172 F T F <--- 8 160-179 92 92 92 170 170 170 F F F 9 180-199 94 93 94 172 171 172 F T F <--- 10 200-219 94 92 94 172 169 172 F T F <--- 11 220-239 93 93 93 171 171 171 T F T <--- 12 240-259 93 93 93 171 171 171 F F F 13 260-279 92 92 92 169 169 169 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 92 92 92 170 170 170 F F F 16 320-339 92 92 92 169 169 169 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 92 92 92 170 170 170 T F T <--- 19 380-399 89 89 89 163 163 163 T F T <--- 20 400-419 93 93 93 171 171 171 F F F 21 420-439 94 90 94 172 166 172 F T F <--- 22 440-459 90 90 90 166 166 166 F F F 23 460-479 92 92 92 170 170 170 T F T <--- 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 92 92 92 170 170 170 T F T <--- 27 540-559 82 82 82 150 150 150 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 94, Max flush: 172 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 173 173 173 F F F 1 020-039 95 95 95 173 173 173 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 95 95 95 173 173 173 F F F 5 100-119 94 94 94 172 172 172 F F F 6 120-139 93 93 93 171 171 171 F F F 7 140-159 94 94 94 172 172 172 F F F 8 160-179 92 92 92 170 170 170 F F F 9 180-199 94 94 94 172 172 172 F F F 10 200-219 94 94 94 172 172 172 F F F 11 220-239 95 95 95 173 173 173 F F F 12 240-259 93 93 93 171 171 171 F F F 13 260-279 92 92 92 169 169 169 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 92 92 92 170 170 170 F F F 16 320-339 92 92 92 169 169 169 F F F 17 340-359 84 84 84 156 156 156 F F F 18 360-379 95 95 95 173 173 173 F F F 19 380-399 95 95 95 173 173 173 F F F 20 400-419 93 93 93 171 171 171 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 90 90 90 166 166 166 F F F 23 460-479 95 95 95 173 173 173 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 95 95 95 173 173 173 F F F 27 540-559 82 82 82 150 150 150 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 95, Max flush: 173 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.29370301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.342075838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.342917266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.342969524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.342985773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.343007632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} {"msg":"Crucible 85b99307-e978-4dd3-8da0-28ceeda28a68 has session id: 35bba3b1-c5a0-4382-ac54-f439eb05af23","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.343532802Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Upstairs opts: Upstairs UUID: 85b99307-e978-4dd3-8da0-28ceeda28a68, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.34358986Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Crucible stats registered with UUID: 85b99307-e978-4dd3-8da0-28ceeda28a68","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.343620579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:17.343646768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.343905118Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.343960876Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.343990405Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344149569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344179108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} {"msg":"85b99307-e978-4dd3-8da0-28ceeda28a68 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344240296Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344260985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"0","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344278344Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344295173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"2","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344311593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344328852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344353991Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"ds_connection connected from Ok(127.0.0.1:49852)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.34465543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"ds_connection connected from Ok(127.0.0.1:63789)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344702338Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:17.344761176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:17.344792614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:17.344828343Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344856812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.344887901Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.345909092Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"0","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.346035637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"2","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.34648939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:17.346578936Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.345313484Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"ds_connection connected from Ok(127.0.0.1:54795)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.345474838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.347280979Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[0]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 172, 171, 172, 170, 172, 172, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.347984393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[0]R generation[0..12]: [95, 95, 91, 95, 95, 94, 93, 94, 92, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348014811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.34804047Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[1]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 172, 171, 172, 170, 172, 172, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348064569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[1]R generation[0..12]: [95, 95, 91, 95, 95, 94, 93, 94, 92, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348099108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348137727Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[2]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 172, 171, 172, 170, 172, 172, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348187865Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[2]R generation[0..12]: [95, 95, 91, 95, 95, 94, 93, 94, 92, 94, 94, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348229753Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348276021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.34830436Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348335039Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348375528Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348413536Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"":"downstairs","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348438785Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348471094Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"0","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348510463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"1","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348543421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"client":"2","":"downstairs","session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.34857466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348609579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"85b99307-e978-4dd3-8da0-28ceeda28a68 is now active with session: 35bba3b1-c5a0-4382-ac54-f439eb05af23","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348648197Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348676406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590,"session_id":"35bba3b1-c5a0-4382-ac54-f439eb05af23"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348719355Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} {"msg":"Activated sub_volume 85b99307-e978-4dd3-8da0-28ceeda28a68","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:27.348750594Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 158 len 32768 data: 7 10 10 11 10 9 8 8 02/30 Write block 543 len 24576 data: 8 8 5 4 3 5 03/30 Read block 129 len 12288 04/30 Read block 39 len 4096 05/30 Flush 06/30 Write block 116 len 36864 data: 8 11 10 10 10 10 10 11 11 07/30 Write block 326 len 16384 data: 5 4 5 5 08/30 Read block 356 len 24576 09/30 Write block 137 len 28672 data: 9 10 9 8 8 9 7 10/30 Read block 467 len 20480 11/30 Read block 329 len 20480 12/30 Read block 43 len 16384 13/30 Write block 252 len 4096 data: 8 14/30 Write block 197 len 36864 data: 8 8 9 12 12 10 9 11 12 15/30 Read block 541 len 8192 16/30 Read block 23 len 12288 17/30 Write block 403 len 4096 data: 10 18/30 Write block 301 len 12288 data: 15 15 12 19/30 Read block 90 len 12288 20/30 Write block 523 len 20480 data: 5 7 8 7 5 21/30 Read block 200 len 32768 22/30 Read block 20 len 8192 23/30 Read block 415 len 40960 24/30 Read block 428 len 36864 25/30 Write block 216 len 16384 data: 7 8 7 4 26/30 Read block 529 len 12288 27/30 Write block 403 len 16384 data: 11 9 10 9 28/30 Write block 336 len 24576 data: 8 10 9 8 10 8 29/30 Read block 133 len 32768 30/30 Read block 397 len 40960 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) None Some(JobId(1004)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:27.370129089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1590} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 173 173 173 F F F 1 020-039 95 95 95 173 173 173 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 95 95 95 173 173 173 F F F 5 100-119 94 94 94 172 172 172 T F T <--- 6 120-139 93 93 93 171 171 171 T F T <--- 7 140-159 96 94 96 174 172 174 T F T <--- 8 160-179 96 92 96 174 170 174 F F F <--- 9 180-199 94 94 94 172 172 172 T F T <--- 10 200-219 94 94 94 172 172 172 T F T <--- 11 220-239 95 95 95 173 173 173 F F F 12 240-259 93 93 93 171 171 171 T F T <--- 13 260-279 92 92 92 169 169 169 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 92 92 92 170 170 170 T F T <--- 16 320-339 92 92 92 169 169 169 T F T <--- 17 340-359 84 84 84 156 156 156 T F T <--- 18 360-379 95 95 95 173 173 173 F F F 19 380-399 95 95 95 173 173 173 F F F 20 400-419 93 93 93 171 171 171 T F T <--- 21 420-439 94 94 94 172 172 172 F F F 22 440-459 90 90 90 166 166 166 F F F 23 460-479 95 95 95 173 173 173 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 95 95 95 173 173 173 T F T <--- 27 540-559 96 82 96 174 150 174 F F F <--- 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 96, Max flush: 174 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 173 173 173 F F F 1 020-039 95 95 95 173 173 173 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 95 95 95 173 173 173 F F F 5 100-119 97 97 97 175 175 175 F F F 6 120-139 97 97 97 175 175 175 F F F 7 140-159 97 97 97 175 175 175 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 97 97 97 175 175 175 F F F 10 200-219 97 97 97 175 175 175 F F F 11 220-239 95 95 95 173 173 173 F F F 12 240-259 97 97 97 175 175 175 F F F 13 260-279 92 92 92 169 169 169 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 97 97 97 175 175 175 F F F 18 360-379 95 95 95 173 173 173 F F F 19 380-399 95 95 95 173 173 173 F F F 20 400-419 97 97 97 175 175 175 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 90 90 90 166 166 166 F F F 23 460-479 95 95 95 173 173 173 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 96 96 96 174 174 174 F F F 28 560-579 90 90 90 166 166 166 F F F 29 580-599 90 90 90 165 165 165 F F F Max gen: 97, Max flush: 175 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.792137002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:31.837146667Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.837642788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.869844381Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.870609872Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.870660681Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.87067967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.870713268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} {"msg":"Crucible 58defe37-ed3a-4c48-b019-df80ad043bf2 has session id: 467aafa1-6798-4fef-82b3-1b860577bca0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871353654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Upstairs opts: Upstairs UUID: 58defe37-ed3a-4c48-b019-df80ad043bf2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871410632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Crucible stats registered with UUID: 58defe37-ed3a-4c48-b019-df80ad043bf2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871434611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:31.87145451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871643843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871710981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871745279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871861105Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871896043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} {"msg":"58defe37-ed3a-4c48-b019-df80ad043bf2 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871929032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871945572Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"0","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.871962591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"1","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.87197921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"2","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.87199584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.872042178Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.872066187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52041)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.872305238Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47456)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.872413644Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41938)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.872458552Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.873485173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"1","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.873525451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"2","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.87356954Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"0","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874139938Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874176856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[0]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 175, 175, 175, 174, 175, 175, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874197406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[0]R generation[0..12]: [95, 95, 91, 95, 95, 97, 97, 97, 96, 97, 97, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874216745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874236824Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[1]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 175, 175, 175, 174, 175, 175, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874256044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[1]R generation[0..12]: [95, 95, 91, 95, 95, 97, 97, 97, 96, 97, 97, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874274513Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874294062Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[2]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 175, 175, 175, 174, 175, 175, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874313061Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[2]R generation[0..12]: [95, 95, 91, 95, 95, 97, 97, 97, 96, 97, 97, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874331171Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.87435147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874378269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Generation requested: 98 >= found:98","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-02-15T00:01:31.874415858Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874443856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874461296Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"":"downstairs","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874478585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874493924Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"0","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874511104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"1","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874527933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"client":"2","":"downstairs","session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874545013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874560992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"58defe37-ed3a-4c48-b019-df80ad043bf2 is now active with session: 467aafa1-6798-4fef-82b3-1b860577bca0","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874579131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874594231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598,"session_id":"467aafa1-6798-4fef-82b3-1b860577bca0"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.87461013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} {"msg":"Activated sub_volume 58defe37-ed3a-4c48-b019-df80ad043bf2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:31.874624039Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 585 len 12288 data: 8 7 8 02/30 Flush 03/30 Write block 153 len 20480 data: 8 6 8 7 6 04/30 Write block 148 len 40960 data: 9 8 8 8 7 9 7 9 8 7 05/30 Write block 207 len 28672 data: 9 9 7 5 5 6 5 06/30 Read block 62 len 28672 07/30 Read block 160 len 12288 08/30 Read block 300 len 20480 09/30 Write block 556 len 20480 data: 8 10 12 14 12 10/30 Write block 171 len 32768 data: 8 9 9 9 8 7 5 6 11/30 Read block 389 len 32768 12/30 Write block 167 len 16384 data: 7 9 5 9 13/30 Write block 471 len 32768 data: 7 7 6 6 4 5 6 5 14/30 Write block 459 len 12288 data: 4 4 4 15/30 Write block 189 len 36864 data: 8 5 5 3 2 4 6 6 9 16/30 Write block 28 len 8192 data: 12 11 17/30 Write block 100 len 32768 data: 9 8 8 8 7 5 5 5 18/30 Write block 261 len 20480 data: 7 6 6 6 6 19/30 Write block 546 len 36864 data: 5 4 6 6 7 8 8 8 9 20/30 Flush 21/30 Write block 137 len 24576 data: 10 11 10 9 9 10 22/30 Read block 422 len 36864 23/30 Flush 24/30 Flush 25/30 Write block 417 len 24576 data: 8 8 9 9 10 10 26/30 Read block 176 len 4096 27/30 Write block 142 len 36864 data: 11 8 7 7 8 7 10 9 9 28/30 Read block 204 len 20480 29/30 Write block 266 len 8192 data: 6 6 30/30 Read block 232 len 4096 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:31.893585107Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1598} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 173 173 173 F F F 1 020-039 95 98 98 173 177 177 F F F <--- 2 040-059 91 91 91 168 168 168 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 95 95 95 173 173 173 F F F 5 100-119 97 98 98 175 177 177 F F F <--- 6 120-139 97 98 98 175 178 178 F F F <--- 7 140-159 97 98 98 175 178 178 F T T <--- 8 160-179 96 98 98 174 177 177 F F F <--- 9 180-199 97 98 98 175 177 177 F F F <--- 10 200-219 97 98 98 175 177 177 F F F <--- 11 220-239 95 95 95 173 173 173 F F F 12 240-259 97 97 97 175 175 175 F F F 13 260-279 92 98 98 169 177 177 F T T <--- 14 280-299 87 87 87 161 161 161 F F F 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 97 97 97 175 175 175 F F F 18 360-379 95 95 95 173 173 173 F F F 19 380-399 95 95 95 173 173 173 F F F 20 400-419 97 97 97 175 175 175 F T T <--- 21 420-439 94 94 94 172 172 172 F T T <--- 22 440-459 90 98 98 166 177 177 F F F <--- 23 460-479 95 98 98 173 177 177 F F F <--- 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 96 98 98 174 177 177 F F F <--- 28 560-579 90 98 98 166 177 177 F F F <--- 29 580-599 90 98 98 165 176 176 F F F <--- Max gen: 98, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 173 173 173 F F F 1 020-039 98 98 98 177 177 177 F F F 2 040-059 91 91 91 168 168 168 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 95 95 95 173 173 173 F F F 5 100-119 98 98 98 177 177 177 F F F 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 98 98 98 177 177 177 F F F 9 180-199 98 98 98 177 177 177 F F F 10 200-219 98 98 98 177 177 177 F F F 11 220-239 95 95 95 173 173 173 F F F 12 240-259 97 97 97 175 175 175 F F F 13 260-279 99 99 99 179 179 179 F F F 14 280-299 87 87 87 161 161 161 F F F 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 97 97 97 175 175 175 F F F 18 360-379 95 95 95 173 173 173 F F F 19 380-399 95 95 95 173 173 173 F F F 20 400-419 99 99 99 179 179 179 F F F 21 420-439 99 99 99 179 179 179 F F F 22 440-459 98 98 98 177 177 177 F F F 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 98 98 98 177 177 177 F F F 28 560-579 98 98 98 177 177 177 F F F 29 580-599 98 98 98 176 176 176 F F F Max gen: 99, Max flush: 179 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.360332675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.407395612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.408623326Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.408677734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.408701643Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.408724212Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} {"msg":"Crucible 3476f81d-abb4-4c2c-bc0f-2c1b25074356 has session id: c1d8b577-b5be-4a82-baed-a75ddef0d4a3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.409450984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Upstairs opts: Upstairs UUID: 3476f81d-abb4-4c2c-bc0f-2c1b25074356, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.409494572Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Crucible stats registered with UUID: 3476f81d-abb4-4c2c-bc0f-2c1b25074356","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.409514232Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:36.409530121Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.409778222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.40983309Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.409863889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41007655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410107339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} {"msg":"3476f81d-abb4-4c2c-bc0f-2c1b25074356 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410231315Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410263713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"0","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410283133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"1","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410313231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"2","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41034055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41036275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410389578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40144)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410458536Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42305)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410599741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47392)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.410647939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.411818364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"1","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41191502Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"0","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41245626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.412571855Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.413841727Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"2","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[0]R flush_numbers[0..12]: [173, 177, 168, 173, 173, 177, 178, 179, 177, 177, 177, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414432755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[0]R generation[0..12]: [95, 98, 91, 95, 95, 98, 98, 99, 98, 98, 98, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414458123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414478723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[1]R flush_numbers[0..12]: [173, 177, 168, 173, 173, 177, 178, 179, 177, 177, 177, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414504822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[1]R generation[0..12]: [95, 98, 91, 95, 95, 98, 98, 99, 98, 98, 98, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414523841Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41454356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[2]R flush_numbers[0..12]: [173, 177, 168, 173, 173, 177, 178, 179, 177, 177, 177, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.41456265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[2]R generation[0..12]: [95, 98, 91, 95, 95, 98, 98, 99, 98, 98, 98, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414581049Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414600558Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414616657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414633657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414649606Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414665846Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"":"downstairs","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414690725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414705854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"0","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414722713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"1","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414739603Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"client":"2","":"downstairs","session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414759062Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414775871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"3476f81d-abb4-4c2c-bc0f-2c1b25074356 is now active with session: c1d8b577-b5be-4a82-baed-a75ddef0d4a3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414793871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.4148092Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606,"session_id":"c1d8b577-b5be-4a82-baed-a75ddef0d4a3"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414834559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} {"msg":"Activated sub_volume 3476f81d-abb4-4c2c-bc0f-2c1b25074356","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:36.414849988Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 8192 data: 7 6 02/30 Write block 555 len 20480 data: 8 9 11 13 15 03/30 Write block 30 len 24576 data: 9 7 8 6 5 4 04/30 Write block 342 len 8192 data: 9 9 05/30 Write block 436 len 36864 data: 11 10 10 11 10 8 7 7 10 06/30 Read block 286 len 36864 07/30 Read block 51 len 40960 08/30 Read block 183 len 40960 09/30 Write block 284 len 24576 data: 4 4 3 3 3 3 10/30 Read block 291 len 40960 11/30 Read block 67 len 8192 12/30 Write block 115 len 16384 data: 6 9 12 11 13/30 Read block 587 len 32768 14/30 Read block 524 len 36864 15/30 Read block 101 len 20480 16/30 Read block 272 len 28672 17/30 Read block 64 len 20480 18/30 Write block 379 len 16384 data: 4 4 6 4 19/30 Write block 16 len 36864 data: 11 12 9 10 12 14 11 10 12 20/30 Write block 24 len 8192 data: 13 11 21/30 Read block 245 len 8192 22/30 Read block 390 len 4096 23/30 Flush 24/30 Write block 375 len 36864 data: 7 7 7 7 5 5 7 5 3 25/30 Write block 86 len 4096 data: 9 26/30 Write block 368 len 12288 data: 9 7 7 27/30 Write block 445 len 4096 data: 12 28/30 Write block 436 len 32768 data: 12 11 11 12 11 9 8 8 29/30 Write block 557 len 4096 data: 12 30/30 Write block 393 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 5 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 30 36 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:36.4329162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1606} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 95 180 180 173 F F F <--- 1 020-039 100 100 98 180 180 177 F F F <--- 2 040-059 100 100 91 180 180 168 F F F <--- 3 060-079 95 95 95 173 173 173 F F F 4 080-099 95 95 95 173 173 173 T T F <--- 5 100-119 100 100 98 180 180 177 F F F <--- 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 98 98 98 177 177 177 F F F 9 180-199 98 98 98 177 177 177 F F F 10 200-219 98 98 98 177 177 177 F F F 11 220-239 95 95 95 173 173 173 F F F 12 240-259 97 97 97 175 175 175 F F F 13 260-279 99 99 99 179 179 179 F F F 14 280-299 100 100 87 180 180 161 F F F <--- 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 100 100 97 180 180 175 F F F <--- 18 360-379 100 100 95 180 180 173 T T F <--- 19 380-399 100 100 95 180 180 173 T T F <--- 20 400-419 99 99 99 179 179 179 F F F 21 420-439 100 100 99 180 180 179 T T F <--- 22 440-459 100 100 98 180 180 177 T T F <--- 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 100 100 98 180 180 177 T T F <--- 28 560-579 98 98 98 177 177 177 F F F 29 580-599 98 98 98 176 176 176 F F F Max gen: 100, Max flush: 180 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 180 180 180 F F F 1 020-039 100 100 100 180 180 180 F F F 2 040-059 100 100 100 180 180 180 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 101 101 101 181 181 181 F F F 5 100-119 100 100 100 180 180 180 F F F 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 98 98 98 177 177 177 F F F 9 180-199 98 98 98 177 177 177 F F F 10 200-219 98 98 98 177 177 177 F F F 11 220-239 95 95 95 173 173 173 F F F 12 240-259 97 97 97 175 175 175 F F F 13 260-279 99 99 99 179 179 179 F F F 14 280-299 100 100 100 180 180 180 F F F 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 100 100 100 180 180 180 F F F 18 360-379 101 101 101 181 181 181 F F F 19 380-399 101 101 101 181 181 181 F F F 20 400-419 99 99 99 179 179 179 F F F 21 420-439 101 101 101 181 181 181 F F F 22 440-459 101 101 101 181 181 181 F F F 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 101 101 101 181 181 181 F F F 28 560-579 98 98 98 177 177 177 F F F 29 580-599 98 98 98 176 176 176 F F F Max gen: 101, Max flush: 181 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.846294741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:40.892569578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.893041731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.925151188Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.92588122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.925935838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.925952757Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.925974396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} {"msg":"Crucible 18599888-3428-4308-92f0-0efddbc15520 has session id: 5fee4010-633d-4452-b48e-4d7a503c7e48","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.926578133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Upstairs opts: Upstairs UUID: 18599888-3428-4308-92f0-0efddbc15520, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.926629871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Crucible stats registered with UUID: 18599888-3428-4308-92f0-0efddbc15520","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.92665548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:40.92667225Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.926894251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.92693113Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.926949489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927139112Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927162241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} {"msg":"18599888-3428-4308-92f0-0efddbc15520 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927232198Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927252557Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"0","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927269577Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"1","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927286236Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"2","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927302646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927319975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927341654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:37743)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927513458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:45087)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927746049Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:45247)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.927791017Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.928885575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"1","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929111877Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"2","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929243701Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"0","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929618647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929697514Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[0]R flush_numbers[0..12]: [180, 180, 180, 173, 181, 180, 178, 179, 177, 177, 177, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.92980418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[0]R generation[0..12]: [100, 100, 100, 95, 101, 100, 98, 99, 98, 98, 98, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929828459Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929848369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[1]R flush_numbers[0..12]: [180, 180, 180, 173, 181, 180, 178, 179, 177, 177, 177, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929867468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[1]R generation[0..12]: [100, 100, 100, 95, 101, 100, 98, 99, 98, 98, 98, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929885797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929905946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[2]R flush_numbers[0..12]: [180, 180, 180, 173, 181, 180, 178, 179, 177, 177, 177, 173]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929924746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[2]R generation[0..12]: [100, 100, 100, 95, 101, 100, 98, 99, 98, 98, 98, 95]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929947635Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.929967204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:01:40.929990993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930025202Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930053281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930083269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"":"downstairs","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930105279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930125498Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"0","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930155027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"1","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930185726Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"client":"2","":"downstairs","session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930221254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930246903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"18599888-3428-4308-92f0-0efddbc15520 is now active with session: 5fee4010-633d-4452-b48e-4d7a503c7e48","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930265733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930280992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614,"session_id":"5fee4010-633d-4452-b48e-4d7a503c7e48"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930297061Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} {"msg":"Activated sub_volume 18599888-3428-4308-92f0-0efddbc15520","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:40.930310931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 368 len 12288 data: 10 8 8 02/30 Read block 461 len 4096 03/30 Read block 98 len 28672 04/30 Read block 312 len 12288 05/30 Read block 287 len 40960 06/30 Read block 330 len 40960 07/30 Read block 178 len 12288 08/30 Read block 166 len 32768 09/30 Read block 559 len 12288 10/30 Read block 173 len 20480 11/30 Write block 163 len 32768 data: 10 9 9 7 8 10 6 10 12/30 Read block 262 len 12288 13/30 Read block 152 len 28672 14/30 Read block 106 len 4096 15/30 Read block 217 len 16384 16/30 Write block 554 len 24576 data: 10 9 10 13 14 16 17/30 Read block 244 len 28672 18/30 Read block 292 len 16384 19/30 Write block 185 len 28672 data: 7 8 6 7 9 6 6 20/30 Write block 259 len 40960 data: 8 6 8 7 7 7 7 7 7 7 21/30 Write block 348 len 4096 data: 10 22/30 Flush 23/30 Write block 185 len 40960 data: 8 9 7 8 10 7 7 4 3 5 24/30 Write block 575 len 4096 data: 15 25/30 Read block 284 len 24576 26/30 Write block 232 len 8192 data: 9 7 27/30 Read block 503 len 12288 28/30 Read block 269 len 36864 29/30 Read block 312 len 32768 30/30 Write block 340 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:40.95054053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1614} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 180 180 180 F F F 1 020-039 100 100 100 180 180 180 F F F 2 040-059 100 100 100 180 180 180 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 101 101 101 181 181 181 F F F 5 100-119 100 100 100 180 180 180 F F F 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 98 102 102 177 182 182 F F F <--- 9 180-199 98 102 102 177 182 182 F T T <--- 10 200-219 98 98 98 177 177 177 F F F 11 220-239 95 95 95 173 173 173 F T T <--- 12 240-259 97 102 102 175 182 182 F F F <--- 13 260-279 99 102 102 179 182 182 F F F <--- 14 280-299 100 100 100 180 180 180 F F F 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 100 102 102 180 182 182 F T T <--- 18 360-379 101 102 102 181 182 182 T F F <--- 19 380-399 101 101 101 181 181 181 F F F 20 400-419 99 99 99 179 179 179 F F F 21 420-439 101 101 101 181 181 181 F F F 22 440-459 101 101 101 181 181 181 F F F 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 101 102 102 181 182 182 F F F <--- 28 560-579 98 98 98 177 177 177 F T T <--- 29 580-599 98 98 98 176 176 176 F F F Max gen: 102, Max flush: 182 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 180 180 180 F F F 1 020-039 100 100 100 180 180 180 F F F 2 040-059 100 100 100 180 180 180 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 101 101 101 181 181 181 F F F 5 100-119 100 100 100 180 180 180 F F F 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 102 102 102 182 182 182 F F F 9 180-199 103 103 103 183 183 183 F F F 10 200-219 98 98 98 177 177 177 F F F 11 220-239 103 103 103 183 183 183 F F F 12 240-259 102 102 102 182 182 182 F F F 13 260-279 102 102 102 182 182 182 F F F 14 280-299 100 100 100 180 180 180 F F F 15 300-319 97 97 97 175 175 175 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 103 103 103 183 183 183 F F F 18 360-379 102 102 102 182 182 182 F F F 19 380-399 101 101 101 181 181 181 F F F 20 400-419 99 99 99 179 179 179 F F F 21 420-439 101 101 101 181 181 181 F F F 22 440-459 101 101 101 181 181 181 F F F 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F F F 26 520-539 97 97 97 175 175 175 F F F 27 540-559 102 102 102 182 182 182 F F F 28 560-579 103 103 103 183 183 183 F F F 29 580-599 98 98 98 176 176 176 F F F Max gen: 103, Max flush: 183 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.415795346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:45.461733806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.462260176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.494882843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496039499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496088107Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496104227Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496126046Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} {"msg":"Crucible 4a4ccbb6-b041-472e-a6e4-1463d8eebaff has session id: 2cc43cfc-5cf6-4c2e-bc88-75109b8d329e","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496621027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Upstairs opts: Upstairs UUID: 4a4ccbb6-b041-472e-a6e4-1463d8eebaff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496687844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Crucible stats registered with UUID: 4a4ccbb6-b041-472e-a6e4-1463d8eebaff","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496714103Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:45.496736052Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.496993593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497043601Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.49706682Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497250593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497277032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} {"msg":"4a4ccbb6-b041-472e-a6e4-1463d8eebaff active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497339469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497360379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"0","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497377638Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"1","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497394538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"2","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497418586Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497450755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497482034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63910)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497516153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40714)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.49759007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44658)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.497635298Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.498865951Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"2","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.49891458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"1","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499045995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"0","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499549175Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499583824Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[0]R flush_numbers[0..12]: [180, 180, 180, 173, 181, 180, 178, 179, 182, 183, 177, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499649432Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[0]R generation[0..12]: [100, 100, 100, 95, 101, 100, 98, 99, 102, 103, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.49967406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.49969453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[1]R flush_numbers[0..12]: [180, 180, 180, 173, 181, 180, 178, 179, 182, 183, 177, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499713329Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[1]R generation[0..12]: [100, 100, 100, 95, 101, 100, 98, 99, 102, 103, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499732028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499751498Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[2]R flush_numbers[0..12]: [180, 180, 180, 173, 181, 180, 178, 179, 182, 183, 177, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499770307Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[2]R generation[0..12]: [100, 100, 100, 95, 101, 100, 98, 99, 102, 103, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499789006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499808315Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499830985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499859844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:01:45.499891332Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499929731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"":"downstairs","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.49995593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499972019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"0","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.499997348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"1","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500027937Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"client":"2","":"downstairs","session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500061256Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500080185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"4a4ccbb6-b041-472e-a6e4-1463d8eebaff is now active with session: 2cc43cfc-5cf6-4c2e-bc88-75109b8d329e","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500104894Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500132093Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622,"session_id":"2cc43cfc-5cf6-4c2e-bc88-75109b8d329e"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500162272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} {"msg":"Activated sub_volume 4a4ccbb6-b041-472e-a6e4-1463d8eebaff","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:45.500179141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 4096 data: 9 02/30 Write block 295 len 36864 data: 10 10 10 13 13 14 16 16 13 03/30 Write block 4 len 36864 data: 6 7 6 7 8 8 8 8 10 04/30 Read block 306 len 16384 05/30 Write block 586 len 4096 data: 8 06/30 Read block 439 len 12288 07/30 Read block 573 len 24576 08/30 Read block 78 len 32768 09/30 Write block 378 len 4096 data: 8 10/30 Read block 273 len 16384 11/30 Write block 439 len 12288 data: 13 12 10 12/30 Write block 20 len 32768 data: 13 15 12 11 14 12 11 10 13/30 Write block 260 len 28672 data: 7 9 8 8 8 8 8 14/30 Write block 406 len 32768 data: 10 9 10 9 9 10 8 7 15/30 Write block 46 len 16384 data: 7 10 11 12 16/30 Write block 169 len 4096 data: 7 17/30 Read block 168 len 40960 18/30 Flush 19/30 Write block 424 len 8192 data: 12 9 20/30 Read block 48 len 32768 21/30 Read block 22 len 20480 22/30 Write block 383 len 32768 data: 4 4 5 4 4 4 4 6 23/30 Read block 322 len 8192 24/30 Write block 175 len 20480 data: 9 8 6 7 6 25/30 Read block 304 len 12288 26/30 Read block 512 len 12288 27/30 Write block 573 len 24576 data: 11 12 16 15 13 11 28/30 Write block 517 len 32768 data: 9 10 7 7 5 7 6 8 29/30 Read block 508 len 16384 30/30 Write block 240 len 28672 data: 6 5 5 5 4 5 7 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:45.518552381Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1622} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 104 104 180 184 184 F F F <--- 1 020-039 100 104 104 180 184 184 F F F <--- 2 040-059 100 104 104 180 184 184 F F F <--- 3 060-079 95 95 95 173 173 173 F F F 4 080-099 101 101 101 181 181 181 F F F 5 100-119 100 104 104 180 184 184 T F F <--- 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 102 104 104 182 184 184 F T T <--- 9 180-199 103 103 103 183 183 183 F F F 10 200-219 98 98 98 177 177 177 F F F 11 220-239 103 103 103 183 183 183 F F F 12 240-259 102 102 102 182 182 182 F T T <--- 13 260-279 102 104 104 182 184 184 F F F <--- 14 280-299 100 104 104 180 184 184 F F F <--- 15 300-319 97 104 104 175 184 184 F F F <--- 16 320-339 97 97 97 175 175 175 F F F 17 340-359 103 103 103 183 183 183 F F F 18 360-379 102 104 104 182 184 184 F F F <--- 19 380-399 101 101 101 181 181 181 F T T <--- 20 400-419 99 104 104 179 184 184 F F F <--- 21 420-439 101 104 104 181 184 184 F T T <--- 22 440-459 101 104 104 181 184 184 F F F <--- 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 89 89 89 163 163 163 F T T <--- 26 520-539 97 97 97 175 175 175 F T T <--- 27 540-559 102 102 102 182 182 182 F F F 28 560-579 103 103 103 183 183 183 F T T <--- 29 580-599 98 104 104 176 184 184 F F F <--- Max gen: 104, Max flush: 184 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 104 104 104 184 184 184 F F F 2 040-059 104 104 104 184 184 184 F F F 3 060-079 95 95 95 173 173 173 F F F 4 080-099 101 101 101 181 181 181 F F F 5 100-119 104 104 104 184 184 184 F F F 6 120-139 98 98 98 178 178 178 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 105 105 105 185 185 185 F F F 9 180-199 103 103 103 183 183 183 F F F 10 200-219 98 98 98 177 177 177 F F F 11 220-239 103 103 103 183 183 183 F F F 12 240-259 105 105 105 185 185 185 F F F 13 260-279 104 104 104 184 184 184 F F F 14 280-299 104 104 104 184 184 184 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 97 97 97 175 175 175 F F F 17 340-359 103 103 103 183 183 183 F F F 18 360-379 104 104 104 184 184 184 F F F 19 380-399 105 105 105 185 185 185 F F F 20 400-419 104 104 104 184 184 184 F F F 21 420-439 105 105 105 185 185 185 F F F 22 440-459 104 104 104 184 184 184 F F F 23 460-479 98 98 98 177 177 177 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 105 105 105 185 185 185 F F F 26 520-539 105 105 105 185 185 185 F F F 27 540-559 102 102 102 182 182 182 F F F 28 560-579 105 105 105 185 185 185 F F F 29 580-599 104 104 104 184 184 184 F F F Max gen: 105, Max flush: 185 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:49.984994411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:50.03015906Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.030593504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.061896072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.062627034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.062678142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.062698551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.06272091Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} {"msg":"Crucible ace8ca1c-9a3b-4ea3-bdc0-3b17f2769bed has session id: 124799d5-ed25-495a-9990-30835fabcc2d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.06325584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Upstairs opts: Upstairs UUID: ace8ca1c-9a3b-4ea3-bdc0-3b17f2769bed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063315108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Crucible stats registered with UUID: ace8ca1c-9a3b-4ea3-bdc0-3b17f2769bed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063345116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:50.063362626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063551519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063587607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063605617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063901345Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.063932804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} {"msg":"ace8ca1c-9a3b-4ea3-bdc0-3b17f2769bed active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064011981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.0640327Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"0","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064050009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"1","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064070939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"2","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064087778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064105127Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064122197Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58724)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064213493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51057)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.064265791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45110)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.0643037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.065573931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"2","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.065656028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"1","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.065756455Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"0","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066121981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066179359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[0]R flush_numbers[0..12]: [184, 184, 184, 173, 181, 184, 178, 179, 185, 183, 177, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066382411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[0]R generation[0..12]: [104, 104, 104, 95, 101, 104, 98, 99, 105, 103, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.06640768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066428109Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[1]R flush_numbers[0..12]: [184, 184, 184, 173, 181, 184, 178, 179, 185, 183, 177, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066451768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[1]R generation[0..12]: [104, 104, 104, 95, 101, 104, 98, 99, 105, 103, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066470947Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066490547Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[2]R flush_numbers[0..12]: [184, 184, 184, 173, 181, 184, 178, 179, 185, 183, 177, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066509276Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[2]R generation[0..12]: [104, 104, 104, 95, 101, 104, 98, 99, 105, 103, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066527625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066547254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066563444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066580383Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:01:50.066596352Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066617162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"":"downstairs","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066633951Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066649181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"0","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.06666607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"1","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066682589Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"client":"2","":"downstairs","session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066703039Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066719328Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"ace8ca1c-9a3b-4ea3-bdc0-3b17f2769bed is now active with session: 124799d5-ed25-495a-9990-30835fabcc2d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066737387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066752577Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630,"session_id":"124799d5-ed25-495a-9990-30835fabcc2d"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066772316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} {"msg":"Activated sub_volume ace8ca1c-9a3b-4ea3-bdc0-3b17f2769bed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:50.066786275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 28672 data: 8 7 9 10 13 10 10 02/30 Write block 118 len 24576 data: 12 11 11 11 11 12 03/30 Write block 579 len 40960 data: 10 10 9 8 8 8 9 9 9 6 04/30 Read block 232 len 36864 05/30 Read block 201 len 20480 06/30 Read block 489 len 4096 07/30 Write block 245 len 24576 data: 6 8 6 6 6 7 08/30 Read block 151 len 40960 09/30 Read block 538 len 40960 10/30 Read block 490 len 28672 11/30 Read block 123 len 40960 12/30 Write block 323 len 4096 data: 9 13/30 Write block 199 len 16384 data: 10 13 13 11 14/30 Read block 175 len 4096 15/30 Write block 571 len 40960 data: 12 10 12 13 17 16 14 12 11 11 16/30 Read block 512 len 8192 17/30 Read block 236 len 12288 18/30 Read block 244 len 24576 19/30 Write block 508 len 8192 data: 6 5 20/30 Read block 285 len 40960 21/30 Write block 461 len 24576 data: 5 7 7 6 6 4 22/30 Read block 440 len 28672 23/30 Flush 24/30 Write block 406 len 36864 data: 11 10 11 10 10 11 9 8 7 25/30 Read block 465 len 28672 26/30 Read block 169 len 16384 27/30 Read block 191 len 20480 28/30 Write block 435 len 28672 data: 13 13 12 12 14 13 11 29/30 Read block 474 len 28672 30/30 Write block 21 len 32768 data: 16 13 12 15 13 12 11 13 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 14 1 1 16 Done 16 29 29 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:50.088332435Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1630} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 104 104 104 184 184 184 F T T <--- 2 040-059 104 106 106 184 186 186 T F F <--- 3 060-079 95 106 106 173 186 186 T F F <--- 4 080-099 101 101 101 181 181 181 F F F 5 100-119 104 106 106 184 186 186 T F F <--- 6 120-139 98 106 106 178 186 186 T F F <--- 7 140-159 99 99 99 179 179 179 F F F 8 160-179 105 105 105 185 185 185 F F F 9 180-199 103 106 106 183 186 186 T F F <--- 10 200-219 98 106 106 177 186 186 T F F <--- 11 220-239 103 103 103 183 183 183 F F F 12 240-259 105 106 106 185 186 186 T F F <--- 13 260-279 104 104 104 184 184 184 F F F 14 280-299 104 104 104 184 184 184 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 97 106 106 175 186 186 T F F <--- 17 340-359 103 103 103 183 183 183 F F F 18 360-379 104 104 104 184 184 184 F F F 19 380-399 105 105 105 185 185 185 F F F 20 400-419 104 104 104 184 184 184 F T T <--- 21 420-439 105 105 105 185 185 185 F T T <--- 22 440-459 104 104 104 184 184 184 F T T <--- 23 460-479 98 106 106 177 186 186 F F F <--- 24 480-499 92 92 92 169 169 169 F F F 25 500-519 105 106 106 185 186 186 F F F <--- 26 520-539 105 105 105 185 185 185 F F F 27 540-559 102 102 102 182 182 182 F F F 28 560-579 105 106 106 185 186 186 T F F <--- 29 580-599 104 106 106 184 186 186 T F F <--- Max gen: 106, Max flush: 186 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 106 106 106 186 186 186 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 101 101 101 181 181 181 F F F 5 100-119 106 106 106 186 186 186 F F F 6 120-139 106 106 106 186 186 186 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 105 105 105 185 185 185 F F F 9 180-199 106 106 106 186 186 186 F F F 10 200-219 106 106 106 186 186 186 F F F 11 220-239 103 103 103 183 183 183 F F F 12 240-259 106 106 106 186 186 186 F F F 13 260-279 104 104 104 184 184 184 F F F 14 280-299 104 104 104 184 184 184 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 106 106 106 186 186 186 F F F 17 340-359 103 103 103 183 183 183 F F F 18 360-379 104 104 104 184 184 184 F F F 19 380-399 105 105 105 185 185 185 F F F 20 400-419 107 107 107 187 187 187 F F F 21 420-439 107 107 107 187 187 187 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 106 106 106 186 186 186 F F F 26 520-539 105 105 105 185 185 185 F F F 27 540-559 102 102 102 182 182 182 F F F 28 560-579 106 106 106 186 186 186 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 107, Max flush: 187 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.575004914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:54.617790974Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.618137301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.648534053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649171829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649219237Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649234766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649256835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} {"msg":"Crucible 9c26d1ea-b680-486a-9ecf-2bc8df4117e6 has session id: bdccf33c-6354-4512-9350-a8d5b378e92f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649738437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Upstairs opts: Upstairs UUID: 9c26d1ea-b680-486a-9ecf-2bc8df4117e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649793305Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Crucible stats registered with UUID: 9c26d1ea-b680-486a-9ecf-2bc8df4117e6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.649828174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:54.649859802Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650091713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650133892Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650153261Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650292966Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650313005Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} {"msg":"9c26d1ea-b680-486a-9ecf-2bc8df4117e6 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650379733Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650399332Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"0","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650423971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"1","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.65044126Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"2","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650474879Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650498768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650516367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37198)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650609614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64461)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.650661972Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57403)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.65071287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.651997611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"1","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652034599Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"2","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652055819Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"0","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652604838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652727213Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[0]R flush_numbers[0..12]: [184, 187, 186, 186, 181, 186, 186, 179, 185, 186, 186, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652760942Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[0]R generation[0..12]: [104, 107, 106, 106, 101, 106, 106, 99, 105, 106, 106, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652781381Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.65280172Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[1]R flush_numbers[0..12]: [184, 187, 186, 186, 181, 186, 186, 179, 185, 186, 186, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.65282072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[1]R generation[0..12]: [104, 107, 106, 106, 101, 106, 106, 99, 105, 106, 106, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652839189Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652858608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[2]R flush_numbers[0..12]: [184, 187, 186, 186, 181, 186, 186, 179, 185, 186, 186, 183]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652877458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[2]R generation[0..12]: [104, 107, 106, 106, 101, 106, 106, 99, 105, 106, 106, 103]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652899837Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652919586Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652936015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652953104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652969224Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.652985143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"":"downstairs","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653001903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653017282Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"0","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653034912Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"1","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Transition from WaitQuorum to Active","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-02-15T00:01:54.653058391Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"client":"2","":"downstairs","session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.65307549Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653091639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"9c26d1ea-b680-486a-9ecf-2bc8df4117e6 is now active with session: bdccf33c-6354-4512-9350-a8d5b378e92f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653109439Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653124458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638,"session_id":"bdccf33c-6354-4512-9350-a8d5b378e92f"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653140337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} {"msg":"Activated sub_volume 9c26d1ea-b680-486a-9ecf-2bc8df4117e6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:54.653154057Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 400 len 32768 data: 8 6 7 12 10 11 12 11 02/30 Write block 516 len 40960 data: 9 10 11 8 8 6 8 7 9 9 03/30 Write block 206 len 32768 data: 10 10 10 8 6 6 7 6 04/30 Write block 372 len 4096 data: 9 05/30 Flush 06/30 Write block 41 len 12288 data: 9 8 9 07/30 Read block 116 len 16384 08/30 Read block 454 len 20480 09/30 Read block 344 len 32768 10/30 Read block 469 len 8192 11/30 Write block 388 len 12288 data: 5 5 7 12/30 Write block 375 len 16384 data: 8 8 8 9 13/30 Write block 274 len 4096 data: 10 14/30 Write block 134 len 20480 data: 6 8 9 11 12 15/30 Write block 239 len 4096 data: 8 16/30 Read block 389 len 24576 17/30 Write block 336 len 40960 data: 9 11 10 9 12 9 10 10 9 9 18/30 Read block 339 len 8192 19/30 Write block 332 len 28672 data: 6 6 4 5 10 12 11 20/30 Read block 226 len 16384 21/30 Flush 22/30 Write block 86 len 32768 data: 10 8 8 9 6 5 6 6 23/30 Write block 141 len 16384 data: 10 12 9 8 24/30 Read block 424 len 40960 25/30 Write block 428 len 4096 data: 9 26/30 Write block 413 len 28672 data: 9 8 6 7 9 9 10 27/30 Write block 4 len 36864 data: 7 8 7 8 9 9 9 9 11 28/30 Read block 85 len 40960 29/30 Read block 455 len 8192 30/30 Read block 12 len 28672 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 2 29 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:54.671588875Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1638} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 184 184 184 F T T <--- 1 020-039 107 107 107 187 187 187 F F F 2 040-059 106 108 108 186 189 189 F F F <--- 3 060-079 106 106 106 186 186 186 F F F 4 080-099 101 101 101 181 181 181 F T T <--- 5 100-119 106 106 106 186 186 186 F F F 6 120-139 106 108 108 186 189 189 F F F <--- 7 140-159 99 99 99 179 179 179 F T T <--- 8 160-179 105 105 105 185 185 185 F F F 9 180-199 106 106 106 186 186 186 F F F 10 200-219 106 108 108 186 188 188 F F F <--- 11 220-239 103 108 108 183 189 189 F F F <--- 12 240-259 106 106 106 186 186 186 F F F 13 260-279 104 108 108 184 189 189 F F F <--- 14 280-299 104 104 104 184 184 184 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 106 108 108 186 189 189 F F F <--- 17 340-359 103 108 108 183 189 189 F F F <--- 18 360-379 104 108 108 184 189 189 F F F <--- 19 380-399 105 108 108 185 189 189 F F F <--- 20 400-419 107 108 108 187 188 188 T T T <--- 21 420-439 107 107 107 187 187 187 F T T <--- 22 440-459 107 107 107 187 187 187 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 106 108 108 186 188 188 T F F <--- 26 520-539 105 108 108 185 188 188 T F F <--- 27 540-559 102 102 102 182 182 182 F F F 28 560-579 106 106 106 186 186 186 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 108, Max flush: 189 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 109 109 109 190 190 190 F F F 5 100-119 106 106 106 186 186 186 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 109 109 109 190 190 190 F F F 8 160-179 105 105 105 185 185 185 F F F 9 180-199 106 106 106 186 186 186 F F F 10 200-219 108 108 108 188 188 188 F F F 11 220-239 108 108 108 189 189 189 F F F 12 240-259 106 106 106 186 186 186 F F F 13 260-279 108 108 108 189 189 189 F F F 14 280-299 104 104 104 184 184 184 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 108 108 108 189 189 189 F F F 17 340-359 108 108 108 189 189 189 F F F 18 360-379 108 108 108 189 189 189 F F F 19 380-399 108 108 108 189 189 189 F F F 20 400-419 109 109 109 190 190 190 F F F 21 420-439 109 109 109 190 190 190 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 108 108 108 188 188 188 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 102 102 102 182 182 182 F F F 28 560-579 106 106 106 186 186 186 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 109, Max flush: 190 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.127881931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:59.171897444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.1722918Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.203811989Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.204590579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.204637467Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.204653007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.204675026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} {"msg":"Crucible dc5a1b09-4ced-47cf-b935-8522f43a49e2 has session id: 1b766caf-895a-4460-bc41-9a9e9624357f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205231845Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Upstairs opts: Upstairs UUID: dc5a1b09-4ced-47cf-b935-8522f43a49e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205271013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Crucible stats registered with UUID: dc5a1b09-4ced-47cf-b935-8522f43a49e2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205289123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:59.205304932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205569402Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.20561122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205638179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205772774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205793413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} {"msg":"dc5a1b09-4ced-47cf-b935-8522f43a49e2 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205857571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.20587964Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"0","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205905909Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"1","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205937578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"2","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.205969857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.206000815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.206043744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60425)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.206080912Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51911)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.206116401Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38685)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.206232736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.20745581Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"2","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.207512198Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"1","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.207581665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"0","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208045378Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208072737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 189, 186, 190, 186, 189, 190, 185, 186, 188, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208220881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 106, 109, 106, 108, 109, 105, 106, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.20824485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208265099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 189, 186, 190, 186, 189, 190, 185, 186, 188, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208284358Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 106, 109, 106, 108, 109, 105, 106, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208303008Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208322607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 189, 186, 190, 186, 189, 190, 185, 186, 188, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208341536Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 106, 109, 106, 108, 109, 105, 106, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208360226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208379465Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208395514Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208412224Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208431233Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208447362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"":"downstairs","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208464162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208479261Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"0","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.20849587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"1","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.20851238Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"client":"2","":"downstairs","session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208529219Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"Wait for a query_work_queue command to finish before sending IO 1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208555348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"dc5a1b09-4ced-47cf-b935-8522f43a49e2 is now active with session: 1b766caf-895a-4460-bc41-9a9e9624357f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208573227Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208588247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646,"session_id":"1b766caf-895a-4460-bc41-9a9e9624357f"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208604196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} {"msg":"Activated sub_volume dc5a1b09-4ced-47cf-b935-8522f43a49e2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:01:59.208618006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 425 len 12288 data: 10 9 11 02/30 Write block 143 len 4096 data: 10 03/30 Write block 454 len 8192 data: 6 5 04/30 Write block 334 len 36864 data: 5 6 11 13 12 10 13 10 11 05/30 Read block 386 len 24576 06/30 Read block 571 len 4096 07/30 Write block 276 len 28672 data: 8 9 7 5 4 4 7 08/30 Read block 398 len 36864 09/30 Write block 209 len 20480 data: 9 7 7 8 7 10/30 Write block 120 len 16384 data: 12 12 12 13 11/30 Write block 516 len 40960 data: 10 11 12 9 9 7 9 8 10 10 12/30 Read block 92 len 36864 13/30 Read block 436 len 40960 14/30 Read block 506 len 28672 15/30 Write block 258 len 40960 data: 8 9 8 10 9 9 9 9 9 8 16/30 Read block 361 len 32768 17/30 Read block 398 len 28672 18/30 Read block 491 len 28672 19/30 Read block 414 len 28672 20/30 Write block 365 len 12288 data: 8 7 7 21/30 Read block 106 len 36864 22/30 Read block 362 len 12288 23/30 Read block 174 len 36864 24/30 Write block 83 len 28672 data: 9 9 9 11 9 9 10 25/30 Write block 404 len 36864 data: 11 12 13 12 12 11 11 12 10 26/30 Read block 360 len 24576 27/30 Read block 100 len 40960 28/30 Read block 427 len 20480 29/30 Read block 572 len 8192 30/30 Write block 172 len 8192 data: 10 10 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 18 1 1 20 Done 12 29 29 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:01:59.228315105Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1646} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 109 109 109 190 190 190 F T T <--- 5 100-119 106 106 106 186 186 186 F F F 6 120-139 108 108 108 189 189 189 T T T 7 140-159 109 109 109 190 190 190 T T T 8 160-179 105 105 105 185 185 185 F T T <--- 9 180-199 106 106 106 186 186 186 F F F 10 200-219 108 108 108 188 188 188 T T T 11 220-239 108 108 108 189 189 189 F F F 12 240-259 106 106 106 186 186 186 F T T <--- 13 260-279 108 108 108 189 189 189 T T T 14 280-299 104 104 104 184 184 184 T T T 15 300-319 104 104 104 184 184 184 F F F 16 320-339 108 108 108 189 189 189 T T T 17 340-359 108 108 108 189 189 189 T T T 18 360-379 108 108 108 189 189 189 F T T <--- 19 380-399 108 108 108 189 189 189 F F F 20 400-419 109 109 109 190 190 190 F T T <--- 21 420-439 109 109 109 190 190 190 T T T 22 440-459 107 107 107 187 187 187 T T T 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 108 108 108 188 188 188 T T T 26 520-539 108 108 108 188 188 188 T T T 27 540-559 102 102 102 182 182 182 F F F 28 560-579 106 106 106 186 186 186 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 109, Max flush: 190 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 55 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 263c263,264 < 7, --- > 8, > 10, 264a266,269 > 9, > 9, > 9, > 9, 266,269d270 < 8, < 8, < 8, < 8, 271d271 < 7, diff found after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 110 110 110 191 191 191 F F F 5 100-119 106 106 106 186 186 186 F F F 6 120-139 110 110 110 191 191 191 F F F 7 140-159 110 110 110 191 191 191 F F F 8 160-179 110 110 110 191 191 191 F F F 9 180-199 106 106 106 186 186 186 F F F 10 200-219 110 110 110 191 191 191 F F F 11 220-239 108 108 108 189 189 189 F F F 12 240-259 110 110 110 191 191 191 F F F 13 260-279 110 110 110 191 191 191 F F F 14 280-299 110 110 110 191 191 191 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 110 110 110 191 191 191 F F F 17 340-359 110 110 110 191 191 191 F F F 18 360-379 110 110 110 191 191 191 F F F 19 380-399 108 108 108 189 189 189 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 110 110 110 191 191 191 F F F 26 520-539 110 110 110 191 191 191 F F F 27 540-559 102 102 102 182 182 182 F F F 28 560-579 106 106 106 186 186 186 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 110, Max flush: 191 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.717939602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:03.763237646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.764052306Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.796584006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.797903316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.797956494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.797974203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.797996122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} {"msg":"Crucible e6b2621b-b8e3-46b6-b10b-9724c85a3a79 has session id: 5278e097-7e57-44a1-9902-89cfed6eb818","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.798597629Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Upstairs opts: Upstairs UUID: e6b2621b-b8e3-46b6-b10b-9724c85a3a79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.798640578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Crucible stats registered with UUID: e6b2621b-b8e3-46b6-b10b-9724c85a3a79","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.798659657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:03.798675586Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.798911467Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.798954776Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.798973825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799155758Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799179037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} {"msg":"e6b2621b-b8e3-46b6-b10b-9724c85a3a79 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799251845Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799276573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"0","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799294543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"1","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799311332Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"2","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799327811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799344971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"ds_connection connected from Ok(127.0.0.1:57769)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799471066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"ds_connection connected from Ok(127.0.0.1:40950)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799516054Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"ds_connection connected from Ok(127.0.0.1:44468)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.799764505Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.800839544Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"2","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.800872133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"1","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.800997878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"0","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.80146424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801563006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 189, 186, 191, 186, 191, 191, 191, 186, 191, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801656943Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 106, 110, 106, 110, 110, 110, 106, 110, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801688592Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801709301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 189, 186, 191, 186, 191, 191, 191, 186, 191, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.80172848Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 106, 110, 106, 110, 110, 110, 106, 110, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801751359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801775968Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 189, 186, 191, 186, 191, 191, 191, 186, 191, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801799517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 106, 110, 106, 110, 110, 110, 106, 110, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801822916Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801847226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801867495Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801888634Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801915003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801935062Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"":"downstairs","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801956021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.801975001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"0","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.8019962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"1","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802020129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"client":"2","":"downstairs","session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802041908Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802062067Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"e6b2621b-b8e3-46b6-b10b-9724c85a3a79 is now active with session: 5278e097-7e57-44a1-9902-89cfed6eb818","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802084466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802103516Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657,"session_id":"5278e097-7e57-44a1-9902-89cfed6eb818"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802123815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} {"msg":"Activated sub_volume e6b2621b-b8e3-46b6-b10b-9724c85a3a79","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:03.802141204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 338 len 16384 data: 13 11 14 11 02/30 Read block 410 len 32768 03/30 Read block 105 len 36864 04/30 Read block 178 len 24576 05/30 Read block 410 len 12288 06/30 Read block 5 len 28672 07/30 Read block 550 len 4096 08/30 Read block 430 len 12288 09/30 Write block 215 len 12288 data: 8 8 9 10/30 Read block 181 len 16384 11/30 Read block 411 len 8192 12/30 Write block 92 len 4096 data: 7 13/30 Read block 549 len 40960 14/30 Read block 170 len 28672 15/30 Read block 498 len 16384 16/30 Read block 106 len 4096 17/30 Read block 350 len 12288 18/30 Read block 198 len 28672 19/30 Flush 20/30 Write block 193 len 36864 data: 4 6 7 7 10 9 11 14 14 21/30 Write block 513 len 4096 data: 5 22/30 Flush 23/30 Write block 388 len 4096 data: 6 24/30 Write block 556 len 24576 data: 11 14 15 17 13 12 25/30 Read block 99 len 32768 26/30 Write block 303 len 4096 data: 14 27/30 Write block 522 len 28672 data: 10 9 11 11 8 6 5 28/30 Write block 287 len 36864 data: 4 4 4 4 4 5 8 8 11 29/30 Write block 286 len 28672 data: 4 5 5 5 5 5 6 30/30 Read block 454 len 40960 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Write 7 Sent Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 0 32 Done 0 28 30 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:03.822267418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1657} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 110 112 112 191 192 192 F F F <--- 5 100-119 106 106 106 186 186 186 F F F 6 120-139 110 110 110 191 191 191 F F F 7 140-159 110 110 110 191 191 191 F F F 8 160-179 110 110 110 191 191 191 F F F 9 180-199 106 112 112 186 193 193 F F F <--- 10 200-219 110 112 112 191 193 193 F F F <--- 11 220-239 108 108 108 189 189 189 F F F 12 240-259 110 110 110 191 191 191 F F F 13 260-279 110 110 110 191 191 191 F F F 14 280-299 110 110 110 191 191 191 F T T <--- 15 300-319 104 104 104 184 184 184 F T T <--- 16 320-339 110 112 112 191 192 192 F F F <--- 17 340-359 110 112 112 191 192 192 F F F <--- 18 360-379 110 110 110 191 191 191 F F F 19 380-399 108 108 108 189 189 189 F T T <--- 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 110 112 112 191 193 193 F F F <--- 26 520-539 110 110 110 191 191 191 F T T <--- 27 540-559 102 102 102 182 182 182 F T T <--- 28 560-579 106 106 106 186 186 186 F T T <--- 29 580-599 106 106 106 186 186 186 F F F Max gen: 112, Max flush: 193 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 112 112 112 192 192 192 F F F 5 100-119 106 106 106 186 186 186 F F F 6 120-139 110 110 110 191 191 191 F F F 7 140-159 110 110 110 191 191 191 F F F 8 160-179 110 110 110 191 191 191 F F F 9 180-199 112 112 112 193 193 193 F F F 10 200-219 112 112 112 193 193 193 F F F 11 220-239 108 108 108 189 189 189 F F F 12 240-259 110 110 110 191 191 191 F F F 13 260-279 110 110 110 191 191 191 F F F 14 280-299 113 113 113 194 194 194 F F F 15 300-319 113 113 113 194 194 194 F F F 16 320-339 112 112 112 192 192 192 F F F 17 340-359 112 112 112 192 192 192 F F F 18 360-379 110 110 110 191 191 191 F F F 19 380-399 113 113 113 194 194 194 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 112 112 112 193 193 193 F F F 26 520-539 113 113 113 194 194 194 F F F 27 540-559 113 113 113 194 194 194 F F F 28 560-579 113 113 113 194 194 194 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 113, Max flush: 194 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.296734972Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.341437609Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342218359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342267527Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342283777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342305896Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} {"msg":"Crucible af4776e5-87ef-45ea-b9d9-4175cd18671c has session id: d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342894904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Upstairs opts: Upstairs UUID: af4776e5-87ef-45ea-b9d9-4175cd18671c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342944062Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Crucible stats registered with UUID: af4776e5-87ef-45ea-b9d9-4175cd18671c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.342969951Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:08.34298687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343196742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343234551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.34325341Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343482361Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.34351162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} {"msg":"af4776e5-87ef-45ea-b9d9-4175cd18671c active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343578718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343598717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"0","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343615646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"1","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343632915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"2","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343649305Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343666284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343683263Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43047)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343871156Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50469)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.343916505Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45032)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.344018731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.34509626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"0","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.345141028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"2","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.345729405Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.345828012Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.346827494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"1","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 189, 186, 192, 186, 191, 191, 191, 193, 193, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347378813Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 106, 112, 106, 110, 110, 110, 112, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347408812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347434031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 189, 186, 192, 186, 191, 191, 191, 193, 193, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.34745764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 106, 112, 106, 110, 110, 110, 112, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347481449Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347506028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 189, 186, 192, 186, 191, 191, 191, 193, 193, 189]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347529587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 106, 112, 106, 110, 110, 110, 112, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347552946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347587725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347608744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347629953Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347646032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347662052Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"":"downstairs","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347678831Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347693921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"0","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.34771058Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"1","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347727229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"client":"2","":"downstairs","session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347744339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347760258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"af4776e5-87ef-45ea-b9d9-4175cd18671c is now active with session: d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347782007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347797367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665,"session_id":"d20d7e9f-9b7a-46d5-aca3-fbacc4d6253a"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347813456Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} {"msg":"Activated sub_volume af4776e5-87ef-45ea-b9d9-4175cd18671c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:08.347827216Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 253 len 40960 data: 9 8 8 8 8 9 10 8 10 9 04/30 Flush 05/30 Write block 93 len 24576 data: 7 7 9 6 7 6 06/30 Read block 406 len 8192 07/30 Write block 137 len 24576 data: 12 13 11 10 11 13 08/30 Read block 78 len 28672 09/30 Read block 427 len 36864 10/30 Read block 260 len 24576 11/30 Flush 12/30 Write block 569 len 32768 data: 9 11 13 11 13 14 18 17 13/30 Read block 397 len 28672 14/30 Write block 223 len 28672 data: 10 9 6 4 5 7 7 15/30 Read block 558 len 40960 16/30 Read block 160 len 4096 17/30 Read block 250 len 4096 18/30 Read block 556 len 4096 19/30 Write block 572 len 12288 data: 12 14 15 20/30 Read block 491 len 28672 21/30 Write block 171 len 40960 data: 9 11 11 10 10 9 7 8 7 4 22/30 Read block 276 len 24576 23/30 Read block 153 len 36864 24/30 Read block 471 len 28672 25/30 Read block 155 len 8192 26/30 Read block 140 len 12288 27/30 Read block 274 len 12288 28/30 Write block 361 len 40960 data: 7 7 8 7 9 8 8 11 9 9 29/30 Read block 284 len 16384 30/30 Read block 465 len 8192 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1000)) Some(JobId(1010)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:08.368991259Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1665} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 114 112 114 198 192 198 F F F <--- 5 100-119 106 106 106 186 186 186 F F F 6 120-139 114 110 114 198 191 198 F F F <--- 7 140-159 114 110 114 198 191 198 F F F <--- 8 160-179 110 110 110 191 191 191 T F T <--- 9 180-199 112 112 112 193 193 193 T F T <--- 10 200-219 112 112 112 193 193 193 F F F 11 220-239 108 108 108 189 189 189 T F T <--- 12 240-259 114 110 114 197 191 197 F F F <--- 13 260-279 114 110 114 197 191 197 F F F <--- 14 280-299 113 113 113 194 194 194 F F F 15 300-319 113 113 113 194 194 194 F F F 16 320-339 112 112 112 192 192 192 F F F 17 340-359 112 112 112 192 192 192 F F F 18 360-379 110 110 110 191 191 191 T F T <--- 19 380-399 113 113 113 194 194 194 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 112 112 112 193 193 193 F F F 26 520-539 113 113 113 194 194 194 F F F 27 540-559 113 113 113 194 194 194 F F F 28 560-579 113 113 113 194 194 194 T F T <--- 29 580-599 106 106 106 186 186 186 F F F Max gen: 114, Max flush: 198 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 114 114 114 198 198 198 F F F 5 100-119 106 106 106 186 186 186 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 114 114 114 198 198 198 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 115 115 115 199 199 199 F F F 10 200-219 112 112 112 193 193 193 F F F 11 220-239 115 115 115 199 199 199 F F F 12 240-259 114 114 114 197 197 197 F F F 13 260-279 114 114 114 197 197 197 F F F 14 280-299 113 113 113 194 194 194 F F F 15 300-319 113 113 113 194 194 194 F F F 16 320-339 112 112 112 192 192 192 F F F 17 340-359 112 112 112 192 192 192 F F F 18 360-379 115 115 115 199 199 199 F F F 19 380-399 113 113 113 194 194 194 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 112 112 112 193 193 193 F F F 26 520-539 113 113 113 194 194 194 F F F 27 540-559 113 113 113 194 194 194 F F F 28 560-579 115 115 115 199 199 199 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 115, Max flush: 199 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.77372924Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:12.819935909Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.820664902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.853198782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.854498103Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.854559871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.8545796Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.854607049Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} {"msg":"Crucible f31fa6d3-73d2-4f19-83b3-2d00946652fe has session id: 59b999a4-12cb-4d72-811a-cf7cabd228b5","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855202366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Upstairs opts: Upstairs UUID: f31fa6d3-73d2-4f19-83b3-2d00946652fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855266894Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Crucible stats registered with UUID: f31fa6d3-73d2-4f19-83b3-2d00946652fe","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855305752Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:12.855334311Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855549723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855593851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855613621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855781194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855806003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} {"msg":"f31fa6d3-73d2-4f19-83b3-2d00946652fe active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855876971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.85589715Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"0","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855916459Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"1","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855933408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"2","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855949928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855967177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.855984046Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36292)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.856126421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57861)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.856173879Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62280)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.856463978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.857570816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"1","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.857795707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"2","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.857937882Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"0","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.85824529Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858331267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 189, 186, 198, 186, 198, 198, 199, 199, 193, 199]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858591247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 106, 114, 106, 114, 114, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858617486Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858637415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 189, 186, 198, 186, 198, 198, 199, 199, 193, 199]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858656565Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 106, 114, 106, 114, 114, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858675564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858695143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 189, 186, 198, 186, 198, 198, 199, 199, 193, 199]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858714152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 106, 114, 106, 114, 114, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858732982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858752551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.85876877Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.85878617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858802229Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858818408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"":"downstairs","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858845967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858861877Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"0","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858878506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"1","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858895356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"client":"2","":"downstairs","session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858912385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858928564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"f31fa6d3-73d2-4f19-83b3-2d00946652fe is now active with session: 59b999a4-12cb-4d72-811a-cf7cabd228b5","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858946654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858961773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673,"session_id":"59b999a4-12cb-4d72-811a-cf7cabd228b5"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858977732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} {"msg":"Activated sub_volume f31fa6d3-73d2-4f19-83b3-2d00946652fe","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:12.858991512Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 285 len 40960 data: 5 5 6 6 6 6 6 7 9 9 02/30 Write block 544 len 20480 data: 9 6 6 5 7 03/30 Read block 137 len 8192 04/30 Read block 593 len 24576 05/30 Write block 242 len 12288 data: 6 6 5 06/30 Read block 96 len 36864 07/30 Read block 57 len 4096 08/30 Read block 525 len 12288 09/30 Write block 250 len 28672 data: 8 8 9 10 9 9 9 10/30 Read block 122 len 12288 11/30 Write block 144 len 4096 data: 9 12/30 Write block 269 len 32768 data: 9 8 9 10 9 11 10 9 13/30 Read block 408 len 24576 14/30 Write block 96 len 36864 data: 7 8 7 7 10 9 9 10 8 15/30 Write block 214 len 4096 data: 6 16/30 Write block 342 len 32768 data: 12 11 10 10 9 10 11 9 17/30 Read block 454 len 28672 18/30 Flush 19/30 Write block 339 len 8192 data: 12 15 20/30 Read block 75 len 16384 21/30 Read block 476 len 40960 22/30 Read block 566 len 12288 23/30 Read block 465 len 40960 24/30 Flush 25/30 Write block 278 len 32768 data: 8 6 5 5 8 7 5 6 26/30 Read block 313 len 16384 27/30 Read block 106 len 40960 28/30 Read block 522 len 20480 29/30 Write block 517 len 20480 data: 12 13 10 10 8 30/30 Write block 110 len 24576 data: 2 4 6 6 6 7 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:12.880268571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1673} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 114 116 116 198 200 200 F F F <--- 5 100-119 106 116 116 186 200 200 F T T <--- 6 120-139 114 114 114 198 198 198 F F F 7 140-159 114 116 116 198 200 200 F F F <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 115 115 115 199 199 199 F F F 10 200-219 112 116 116 193 200 200 F F F <--- 11 220-239 115 115 115 199 199 199 F F F 12 240-259 114 116 116 197 200 200 F F F <--- 13 260-279 114 116 116 197 200 200 F T T <--- 14 280-299 113 116 116 194 200 200 T T T <--- 15 300-319 113 113 113 194 194 194 F F F 16 320-339 112 116 116 192 201 201 F F F <--- 17 340-359 112 116 116 192 201 201 F F F <--- 18 360-379 115 115 115 199 199 199 F F F 19 380-399 113 113 113 194 194 194 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 112 112 112 193 193 193 F T T <--- 26 520-539 113 113 113 194 194 194 F T T <--- 27 540-559 113 116 116 194 200 200 T F F <--- 28 560-579 115 115 115 199 199 199 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 116, Max flush: 201 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 190 190 190 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 116 116 116 200 200 200 F F F 5 100-119 117 117 117 202 202 202 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 116 116 116 200 200 200 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 115 115 115 199 199 199 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 115 115 115 199 199 199 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 117 117 117 202 202 202 F F F 14 280-299 117 117 117 202 202 202 F F F 15 300-319 113 113 113 194 194 194 F F F 16 320-339 116 116 116 201 201 201 F F F 17 340-359 116 116 116 201 201 201 F F F 18 360-379 115 115 115 199 199 199 F F F 19 380-399 113 113 113 194 194 194 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 117 117 117 202 202 202 F F F 26 520-539 117 117 117 202 202 202 F F F 27 540-559 116 116 116 200 200 200 F F F 28 560-579 115 115 115 199 199 199 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 117, Max flush: 202 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.373554059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:17.422179856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.422931608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.455359412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.456670222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.45673395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.456767679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.456796068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} {"msg":"Crucible ad4df26e-53f6-4c06-9eee-5b2e9a67aff2 has session id: 6c624dec-5d6e-4981-95fb-bc7e121b9b6a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.457437473Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Upstairs opts: Upstairs UUID: ad4df26e-53f6-4c06-9eee-5b2e9a67aff2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.457488731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Crucible stats registered with UUID: ad4df26e-53f6-4c06-9eee-5b2e9a67aff2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.45752366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:17.457545709Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.457745192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.45778635Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.457805149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.457954584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.457975293Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} {"msg":"ad4df26e-53f6-4c06-9eee-5b2e9a67aff2 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.45805059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458072209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"0","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458089608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"1","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458106708Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"2","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458142376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458169075Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458187185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63778)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.45832169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47576)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458372728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48412)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.458636287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.459677548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"1","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.459768995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"2","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460008475Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"0","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460376331Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.46041845Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[0]R flush_numbers[0..12]: [190, 187, 189, 186, 200, 202, 198, 200, 199, 199, 200, 199]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460643711Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 106, 116, 117, 114, 116, 115, 115, 116, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.46067194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460692409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[1]R flush_numbers[0..12]: [190, 187, 189, 186, 200, 202, 198, 200, 199, 199, 200, 199]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460711788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 106, 116, 117, 114, 116, 115, 115, 116, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460739267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460771006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[2]R flush_numbers[0..12]: [190, 187, 189, 186, 200, 202, 198, 200, 199, 199, 200, 199]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460803495Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 106, 116, 117, 114, 116, 115, 115, 116, 115]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460832834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460865803Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460896292Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460915671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.46093792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"}Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.460972289Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"":"downstairs","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461003048Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461023557Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"0","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461040946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"1","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461057485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"client":"2","":"downstairs","session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461074615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461090574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"ad4df26e-53f6-4c06-9eee-5b2e9a67aff2 is now active with session: 6c624dec-5d6e-4981-95fb-bc7e121b9b6a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461112983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461128563Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681,"session_id":"6c624dec-5d6e-4981-95fb-bc7e121b9b6a"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461144562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} {"msg":"Activated sub_volume ad4df26e-53f6-4c06-9eee-5b2e9a67aff2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:17.461158322Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 62 len 28672 data: 11 8 8 7 8 8 6 02/30 Read block 145 len 4096 03/30 Read block 251 len 12288 04/30 Read block 230 len 12288 05/30 Write block 0 len 20480 data: 5 7 6 6 8 06/30 Read block 434 len 32768 07/30 Read block 417 len 20480 08/30 Write block 44 len 24576 data: 9 7 8 11 12 13 09/30 Write block 385 len 20480 data: 6 5 5 7 6 10/30 Read block 319 len 24576 11/30 Read block 189 len 32768 12/30 Write block 229 len 36864 data: 8 9 10 10 8 9 10 10 10 13/30 Read block 149 len 32768 14/30 Write block 261 len 16384 data: 11 10 9 9 15/30 Write block 220 len 4096 data: 5 16/30 Read block 131 len 8192 17/30 Read block 25 len 28672 18/30 Write block 562 len 8192 data: 13 11 19/30 Write block 546 len 32768 data: 7 6 8 7 8 9 9 9 20/30 Write block 392 len 28672 data: 7 10 10 11 12 11 10 21/30 Read block 571 len 28672 22/30 Read block 437 len 24576 23/30 Read block 132 len 16384 24/30 Flush 25/30 Write block 113 len 20480 data: 7 7 8 10 13 26/30 Read block 219 len 8192 27/30 Write block 512 len 20480 data: 3 6 6 8 11 28/30 Read block 265 len 40960 29/30 Write block 291 len 36864 data: 7 8 10 10 12 11 11 14 14 30/30 Read block 81 len 40960 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 0 0 23 Done 7 30 30 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:17.483176823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1681} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 118 118 190 203 203 T F F <--- 1 020-039 107 107 107 187 187 187 F F F 2 040-059 108 118 118 189 203 203 F F F <--- 3 060-079 106 118 118 186 203 203 T F F <--- 4 080-099 116 116 116 200 200 200 F F F 5 100-119 117 117 117 202 202 202 F T T <--- 6 120-139 114 114 114 198 198 198 F F F 7 140-159 116 116 116 200 200 200 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 115 115 115 199 199 199 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 115 118 118 199 203 203 F F F <--- 12 240-259 116 116 116 200 200 200 F F F 13 260-279 117 118 118 202 203 203 F F F <--- 14 280-299 117 117 117 202 202 202 F T T <--- 15 300-319 113 113 113 194 194 194 F F F 16 320-339 116 116 116 201 201 201 F F F 17 340-359 116 116 116 201 201 201 F F F 18 360-379 115 115 115 199 199 199 F F F 19 380-399 113 118 118 194 203 203 F F F <--- 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 117 117 117 202 202 202 F T T <--- 26 520-539 117 117 117 202 202 202 F F F 27 540-559 116 118 118 200 203 203 F F F <--- 28 560-579 115 118 118 199 203 203 F F F <--- 29 580-599 106 106 106 186 186 186 F F F Max gen: 118, Max flush: 203 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 203 203 203 F F F 1 020-039 107 107 107 187 187 187 F F F 2 040-059 118 118 118 203 203 203 F F F 3 060-079 118 118 118 203 203 203 F F F 4 080-099 116 116 116 200 200 200 F F F 5 100-119 119 119 119 204 204 204 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 116 116 116 200 200 200 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 115 115 115 199 199 199 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 118 118 118 203 203 203 F F F 14 280-299 119 119 119 204 204 204 F F F 15 300-319 113 113 113 194 194 194 F F F 16 320-339 116 116 116 201 201 201 F F F 17 340-359 116 116 116 201 201 201 F F F 18 360-379 115 115 115 199 199 199 F F F 19 380-399 118 118 118 203 203 203 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 110 110 110 191 191 191 F F F 23 460-479 106 106 106 186 186 186 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 119 119 119 204 204 204 F F F 26 520-539 117 117 117 202 202 202 F F F 27 540-559 118 118 118 203 203 203 F F F 28 560-579 118 118 118 203 203 203 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 119, Max flush: 204 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:21.965415191Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.011571392Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.012592273Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.012647042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.012664331Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.01268646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} {"msg":"Crucible cb340f03-1a70-4196-a2a0-b41e8b4e26d1 has session id: cd9c0198-591b-476c-9b87-2a9f1c5a61e9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013346805Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Upstairs opts: Upstairs UUID: cb340f03-1a70-4196-a2a0-b41e8b4e26d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013389523Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Crucible stats registered with UUID: cb340f03-1a70-4196-a2a0-b41e8b4e26d1","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013408273Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:22.013424222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013647763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013702931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.01372442Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013930413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.013957552Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} {"msg":"cb340f03-1a70-4196-a2a0-b41e8b4e26d1 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014038869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014059148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"0","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014082337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"1","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014111486Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014145314Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014179043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014212172Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38335)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014305758Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:22.014364736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56720)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014410584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:22.01453811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:22.014636036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014680764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.014709173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.015852099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"0","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.015957925Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"1","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.016554702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:22.01663583Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.015918577Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55193)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.01610855Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.017598013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[0]R flush_numbers[0..12]: [203, 187, 203, 203, 200, 204, 198, 200, 199, 199, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018288937Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[0]R generation[0..12]: [118, 107, 118, 118, 116, 119, 114, 116, 115, 115, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018327655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018354044Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[1]R flush_numbers[0..12]: [203, 187, 203, 203, 200, 204, 198, 200, 199, 199, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018378583Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[1]R generation[0..12]: [118, 107, 118, 118, 116, 119, 114, 116, 115, 115, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018408002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018432921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[2]R flush_numbers[0..12]: [203, 187, 203, 203, 200, 204, 198, 200, 199, 199, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.01845633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[2]R generation[0..12]: [118, 107, 118, 118, 116, 119, 114, 116, 115, 115, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018480019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018504609Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018524888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018546187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018566066Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018593815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"":"downstairs","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018614774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018633804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"0","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018654783Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"1","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018675822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"client":"2","":"downstairs","session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018697181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018717541Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"cb340f03-1a70-4196-a2a0-b41e8b4e26d1 is now active with session: cd9c0198-591b-476c-9b87-2a9f1c5a61e9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.01874244Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018761739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689,"session_id":"cd9c0198-591b-476c-9b87-2a9f1c5a61e9"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018781838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} {"msg":"Activated sub_volume cb340f03-1a70-4196-a2a0-b41e8b4e26d1","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:32.018799317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 188 len 12288 data: 9 11 8 02/30 Read block 590 len 32768 03/30 Write block 96 len 40960 data: 8 9 8 8 11 10 10 11 9 6 04/30 Read block 147 len 20480 05/30 Read block 85 len 8192 06/30 Write block 156 len 12288 data: 9 8 8 07/30 Read block 443 len 8192 08/30 Write block 110 len 36864 data: 3 5 7 8 8 9 11 14 13 09/30 Read block 573 len 24576 10/30 Write block 100 len 8192 data: 12 11 11/30 Write block 568 len 24576 data: 8 10 12 14 13 15 12/30 Read block 279 len 12288 13/30 Write block 38 len 24576 data: 7 7 9 10 9 10 14/30 Write block 458 len 20480 data: 5 5 5 6 8 15/30 Write block 315 len 28672 data: 10 7 9 10 12 12 13 16/30 Write block 333 len 40960 data: 7 6 7 12 14 14 13 16 12 13 17/30 Read block 381 len 12288 18/30 Write block 302 len 32768 data: 17 15 12 14 14 15 13 15 19/30 Write block 65 len 20480 data: 8 9 9 7 7 20/30 Write block 85 len 16384 data: 10 12 10 10 21/30 Read block 10 len 32768 22/30 Read block 101 len 8192 23/30 Read block 543 len 40960 24/30 Write block 457 len 4096 data: 6 25/30 Flush 26/30 Write block 147 len 36864 data: 8 11 10 10 9 8 10 8 10 27/30 Read block 94 len 8192 28/30 Write block 530 len 40960 data: 8 8 8 8 10 10 11 11 11 10 29/30 Read block 301 len 8192 30/30 Read block 339 len 36864 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 26 27 Done 30 29 4 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:32.040063027Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1689} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 203 203 203 F F F 1 020-039 120 120 107 205 205 187 F F F <--- 2 040-059 120 120 118 205 205 203 F F F <--- 3 060-079 120 120 118 205 205 203 F F F <--- 4 080-099 120 120 116 205 205 200 F F T <--- 5 100-119 120 120 119 205 205 204 F F T <--- 6 120-139 114 114 114 198 198 198 F F F 7 140-159 120 120 116 205 205 200 T T F <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 120 120 115 205 205 199 F F T <--- 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 118 118 118 203 203 203 F F F 14 280-299 119 119 119 204 204 204 F F F 15 300-319 120 120 113 205 205 194 F F F <--- 16 320-339 120 120 116 205 205 201 F F F <--- 17 340-359 120 120 116 205 205 201 F F F <--- 18 360-379 115 115 115 199 199 199 F F F 19 380-399 118 118 118 203 203 203 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 120 120 110 205 205 191 F F F <--- 23 460-479 120 120 106 205 205 186 F F F <--- 24 480-499 92 92 92 169 169 169 F F F 25 500-519 119 119 119 204 204 204 F F F 26 520-539 117 117 117 202 202 202 T T F <--- 27 540-559 118 118 118 203 203 203 F F F 28 560-579 120 120 118 205 205 203 F F F <--- 29 580-599 106 106 106 186 186 186 F F F Max gen: 120, Max flush: 205 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 203 203 203 F F F 1 020-039 120 120 120 205 205 205 F F F 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 120 120 120 205 205 205 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 121 121 121 206 206 206 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 120 120 120 205 205 205 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 118 118 118 203 203 203 F F F 14 280-299 119 119 119 204 204 204 F F F 15 300-319 120 120 120 205 205 205 F F F 16 320-339 120 120 120 205 205 205 F F F 17 340-359 120 120 120 205 205 205 F F F 18 360-379 115 115 115 199 199 199 F F F 19 380-399 118 118 118 203 203 203 F F F 20 400-419 110 110 110 191 191 191 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 92 92 92 169 169 169 F F F 25 500-519 119 119 119 204 204 204 F F F 26 520-539 121 121 121 206 206 206 F F F 27 540-559 118 118 118 203 203 203 F F F 28 560-579 120 120 120 205 205 205 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 121, Max flush: 206 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.474334003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:36.519709774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.520221765Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.551638448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.552547314Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.552596632Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.552612231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.5526341Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} {"msg":"Crucible c666453c-2d71-4486-9f04-9c1480ba2c6d has session id: 5a95036f-ee44-46f7-aa79-090b09686633","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553198699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Upstairs opts: Upstairs UUID: c666453c-2d71-4486-9f04-9c1480ba2c6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553254567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Crucible stats registered with UUID: c666453c-2d71-4486-9f04-9c1480ba2c6d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553289435Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:36.553315754Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553523096Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553574684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553599134Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553741878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553766197Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} {"msg":"c666453c-2d71-4486-9f04-9c1480ba2c6d active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553829965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553855814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"0","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553884493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"1","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553916202Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"2","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.55394809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.553992599Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.554025687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"ds_connection connected from Ok(127.0.0.1:62296)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.554056246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"ds_connection connected from Ok(127.0.0.1:38550)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.554087745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"ds_connection connected from Ok(127.0.0.1:39305)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.554124174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.555392295Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"1","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.555463843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"2","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.55551593Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"0","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556032301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.55606199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[0]R flush_numbers[0..12]: [203, 205, 205, 205, 205, 205, 198, 206, 199, 205, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556087679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[0]R generation[0..12]: [118, 120, 120, 120, 120, 120, 114, 121, 115, 120, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556107838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556127687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[1]R flush_numbers[0..12]: [203, 205, 205, 205, 205, 205, 198, 206, 199, 205, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556146557Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[1]R generation[0..12]: [118, 120, 120, 120, 120, 120, 114, 121, 115, 120, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556165256Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556184955Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[2]R flush_numbers[0..12]: [203, 205, 205, 205, 205, 205, 198, 206, 199, 205, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556203734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[2]R generation[0..12]: [118, 120, 120, 120, 120, 120, 114, 121, 115, 120, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556222423Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556245223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556261712Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556278421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556294501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.55631055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"":"downstairs","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.55632721Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556342559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":Wait for a query_work_queue command to finish before sending IO "0","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556369088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"1","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556385747Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"client":"2","":"downstairs","session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556402787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556418666Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"c666453c-2d71-4486-9f04-9c1480ba2c6d is now active with session: 5a95036f-ee44-46f7-aa79-090b09686633","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556436545Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556451655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697,"session_id":"5a95036f-ee44-46f7-aa79-090b09686633"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556467574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} {"msg":"Activated sub_volume c666453c-2d71-4486-9f04-9c1480ba2c6d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:36.556481374Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 32768 data: 11 11 9 11 10 12 12 9 02/30 Read block 476 len 28672 03/30 Write block 399 len 12288 data: 9 9 7 04/30 Write block 1 len 16384 data: 8 7 7 9 05/30 Read block 497 len 4096 06/30 Read block 512 len 20480 07/30 Read block 471 len 12288 08/30 Write block 414 len 4096 data: 9 09/30 Read block 118 len 12288 10/30 Write block 31 len 16384 data: 8 9 7 6 11/30 Read block 492 len 8192 12/30 Read block 140 len 4096 13/30 Read block 321 len 4096 14/30 Read block 284 len 8192 15/30 Read block 463 len 16384 16/30 Write block 487 len 24576 data: 15 15 14 15 14 13 17/30 Write block 490 len 28672 data: 16 15 14 11 9 9 10 18/30 Read block 144 len 8192 19/30 Write block 143 len 4096 data: 11 20/30 Write block 267 len 40960 data: 8 8 10 9 10 11 10 12 11 10 21/30 Read block 293 len 40960 22/30 Write block 510 len 12288 data: 5 5 4 23/30 Write block 330 len 8192 data: 6 6 24/30 Read block 164 len 16384 25/30 Read block 175 len 32768 26/30 Read block 526 len 12288 27/30 Read block 529 len 24576 28/30 Read block 95 len 36864 29/30 Write block 372 len 20480 data: 10 9 7 9 9 30/30 Write block 319 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:36.574503657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1697} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 203 203 203 F T T <--- 1 020-039 120 120 120 205 205 205 F T T <--- 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 120 120 120 205 205 205 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 121 121 121 206 206 206 F T T <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 120 120 120 205 205 205 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 118 118 118 203 203 203 F T T <--- 14 280-299 119 119 119 204 204 204 F F F 15 300-319 120 120 120 205 205 205 F T T <--- 16 320-339 120 120 120 205 205 205 F T T <--- 17 340-359 120 120 120 205 205 205 F F F 18 360-379 115 115 115 199 199 199 F T T <--- 19 380-399 118 118 118 203 203 203 F T T <--- 20 400-419 110 110 110 191 191 191 F T T <--- 21 420-439 110 110 110 191 191 191 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 92 92 92 169 169 169 F T T <--- 25 500-519 119 119 119 204 204 204 T T T 26 520-539 121 121 121 206 206 206 T T T 27 540-559 118 118 118 203 203 203 F F F 28 560-579 120 120 120 205 205 205 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 121, Max flush: 206 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 61 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 512a513,514 > 5, > 5, 514,515d515 < 4, < 3, diff found after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 207 207 207 F F F 1 020-039 122 122 122 207 207 207 F F F 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 120 120 120 205 205 205 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 122 122 122 207 207 207 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 120 120 120 205 205 205 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 122 122 122 207 207 207 F F F 14 280-299 119 119 119 204 204 204 F F F 15 300-319 122 122 122 207 207 207 F F F 16 320-339 122 122 122 207 207 207 F F F 17 340-359 120 120 120 205 205 205 F F F 18 360-379 122 122 122 207 207 207 F F F 19 380-399 122 122 122 207 207 207 F F F 20 400-419 122 122 122 207 207 207 F F F 21 420-439 110 110 110 191 191 191 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 122 122 122 207 207 207 F F F 25 500-519 122 122 122 207 207 207 F F F 26 520-539 122 122 122 207 207 207 F F F 27 540-559 118 118 118 203 203 203 F F F 28 560-579 120 120 120 205 205 205 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 122, Max flush: 207 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.051430637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:41.095798517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.096329857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.127759679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.128611587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.128673145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.128692844Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.128716613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} {"msg":"Crucible b4aaed07-1da1-48fd-88cb-8cda32516ae6 has session id: a4458cf9-f313-4561-bf9d-dc002364e691","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129289661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Upstairs opts: Upstairs UUID: b4aaed07-1da1-48fd-88cb-8cda32516ae6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.12933801Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Crucible stats registered with UUID: b4aaed07-1da1-48fd-88cb-8cda32516ae6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129365438Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:41.129382338Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.12959986Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129640888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129660087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129798782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129820491Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} {"msg":"b4aaed07-1da1-48fd-88cb-8cda32516ae6 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129883199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129903808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"0","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129928387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"1","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129958626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"2","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.129989854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.130020913Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.130051222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"ds_connection connected from Ok(127.0.0.1:48035)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.130082641Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"ds_connection connected from Ok(127.0.0.1:59194)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.130269874Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"ds_connection connected from Ok(127.0.0.1:64594)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.130444627Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.131349163Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"2","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.131441709Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"1","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.131821265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"0","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132015137Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132045786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[0]R flush_numbers[0..12]: [207, 207, 205, 205, 205, 205, 198, 207, 199, 205, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132435241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[0]R generation[0..12]: [122, 122, 120, 120, 120, 120, 114, 122, 115, 120, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.1324605Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.13248089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[1]R flush_numbers[0..12]: [207, 207, 205, 205, 205, 205, 198, 207, 199, 205, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132500089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[1]R generation[0..12]: [122, 122, 120, 120, 120, 120, 114, 122, 115, 120, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132518828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132538237Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[2]R flush_numbers[0..12]: [207, 207, 205, 205, 205, 205, 198, 207, 199, 205, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132557087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[2]R generation[0..12]: [122, 122, 120, 120, 120, 120, 114, 122, 115, 120, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132576026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132595395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Max found gen is 123","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132611755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Generation requested: 124 >= found:123","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132628654Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132653583Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132673422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"":"downstairs","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132691002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132706541Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"0","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.13272321Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"1","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.13273963Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"client":"2","":"downstairs","session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132756779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132772608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"b4aaed07-1da1-48fd-88cb-8cda32516ae6 is now active with session: a4458cf9-f313-4561-bf9d-dc002364e691","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132790258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132805247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708,"session_id":"a4458cf9-f313-4561-bf9d-dc002364e691"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132821057Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} {"msg":"Activated sub_volume b4aaed07-1da1-48fd-88cb-8cda32516ae6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:41.132834646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 370 len 24576 data: 10 9 11 10 8 10 02/30 Write block 498 len 20480 data: 8 6 7 7 12 03/30 Read block 277 len 4096 04/30 Flush 05/30 Write block 105 len 36864 data: 7 6 6 4 4 4 6 8 9 06/30 Flush 07/30 Write block 421 len 40960 data: 11 11 9 13 11 10 12 10 9 8 08/30 Write block 113 len 40960 data: 10 9 10 12 15 14 12 13 13 13 09/30 Write block 290 len 40960 data: 7 8 9 11 11 13 12 12 15 15 10/30 Write block 579 len 32768 data: 12 12 10 9 9 9 10 10 11/30 Read block 522 len 12288 12/30 Write block 184 len 40960 data: 5 9 10 8 10 12 9 8 5 5 13/30 Flush 14/30 Write block 422 len 28672 data: 12 10 14 12 11 13 11 15/30 Write block 28 len 4096 data: 14 16/30 Read block 517 len 28672 17/30 Read block 139 len 24576 18/30 Flush 19/30 Flush 20/30 Write block 583 len 36864 data: 10 10 11 11 10 7 6 7 6 21/30 Read block 443 len 8192 22/30 Write block 242 len 24576 data: 7 7 6 7 9 7 23/30 Write block 553 len 20480 data: 10 11 10 12 15 24/30 Flush 25/30 Flush 26/30 Write block 385 len 36864 data: 7 6 6 8 7 8 7 8 11 27/30 Read block 94 len 36864 28/30 Write block 368 len 4096 data: 12 29/30 Read block 96 len 4096 30/30 Write block 250 len 16384 data: 9 9 10 11 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:41.153293827Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1708} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 207 207 207 F F F 1 020-039 122 124 124 207 211 211 F F F <--- 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 120 124 124 205 210 210 F F F <--- 6 120-139 114 124 124 198 210 210 F F F <--- 7 140-159 122 122 122 207 207 207 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 120 124 124 205 210 210 F F F <--- 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 116 124 124 200 213 213 F T T <--- 13 260-279 122 122 122 207 207 207 F F F 14 280-299 119 124 124 204 210 210 F F F <--- 15 300-319 122 122 122 207 207 207 F F F 16 320-339 122 122 122 207 207 207 F F F 17 340-359 120 120 120 205 205 205 F F F 18 360-379 122 124 124 207 208 208 T T T <--- 19 380-399 122 122 122 207 207 207 F T T <--- 20 400-419 122 122 122 207 207 207 F F F 21 420-439 110 124 124 191 211 211 F F F <--- 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 122 124 124 207 208 208 T F F <--- 25 500-519 122 124 124 207 208 208 T F F <--- 26 520-539 122 122 122 207 207 207 F F F 27 540-559 118 124 124 203 213 213 F F F <--- 28 560-579 120 124 124 205 210 210 F F F <--- 29 580-599 106 124 124 186 213 213 F F F <--- Max gen: 124, Max flush: 213 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 207 207 207 F F F 1 020-039 124 124 124 211 211 211 F F F 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 124 124 124 210 210 210 F F F 6 120-139 124 124 124 210 210 210 F F F 7 140-159 122 122 122 207 207 207 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 124 124 124 210 210 210 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 118 118 118 203 203 203 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 122 122 122 207 207 207 F F F 14 280-299 124 124 124 210 210 210 F F F 15 300-319 122 122 122 207 207 207 F F F 16 320-339 122 122 122 207 207 207 F F F 17 340-359 120 120 120 205 205 205 F F F 18 360-379 125 125 125 214 214 214 F F F 19 380-399 125 125 125 214 214 214 F F F 20 400-419 122 122 122 207 207 207 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 122 122 122 207 207 207 F F F 27 540-559 124 124 124 213 213 213 F F F 28 560-579 124 124 124 210 210 210 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 125, Max flush: 214 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.659429055Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.707446296Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.708544674Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.708605302Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.708626021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.70865866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} {"msg":"Crucible 2332f375-a4dc-4d9d-8b8c-ceac2ecd0d0f has session id: 4fcba677-10e8-484d-b3ce-3c41e89c75bd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.709260487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Upstairs opts: Upstairs UUID: 2332f375-a4dc-4d9d-8b8c-ceac2ecd0d0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.709313915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Crucible stats registered with UUID: 2332f375-a4dc-4d9d-8b8c-ceac2ecd0d0f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.709337314Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:45.709363523Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.709656612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.70970207Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.709722689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.709988109Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710058616Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} {"msg":"2332f375-a4dc-4d9d-8b8c-ceac2ecd0d0f active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710088745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710114394Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"0","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710133254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"1","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710150793Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"2","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710175082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710209351Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710242669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60549)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710318246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52287)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710506489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:54292)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.710578277Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.711683444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"1","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.711776441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"0","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.711850318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"2","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712276532Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712363298Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[0]R flush_numbers[0..12]: [207, 211, 205, 205, 205, 210, 210, 207, 199, 210, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712474674Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[0]R generation[0..12]: [122, 124, 120, 120, 120, 124, 124, 122, 115, 124, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712498883Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712518692Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[1]R flush_numbers[0..12]: [207, 211, 205, 205, 205, 210, 210, 207, 199, 210, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712537622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[1]R generation[0..12]: [122, 124, 120, 120, 120, 124, 124, 122, 115, 124, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712561281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.71258149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[2]R flush_numbers[0..12]: [207, 211, 205, 205, 205, 210, 210, 207, 199, 210, 200, 203]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712600889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[2]R generation[0..12]: [122, 124, 120, 120, 120, 124, 124, 122, 115, 124, 116, 118]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712619719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712639288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712663417Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712698036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712724885Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712754983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"":"downstairs","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712787562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712817281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"0","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.71285021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"1","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2026-02-15T00:02:45.712887469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"client":"2","":"downstairs","session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712912088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712928787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"2332f375-a4dc-4d9d-8b8c-ceac2ecd0d0f is now active with session: 4fcba677-10e8-484d-b3ce-3c41e89c75bd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712949936Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712965435Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716,"session_id":"4fcba677-10e8-484d-b3ce-3c41e89c75bd"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712982825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} {"msg":"Activated sub_volume 2332f375-a4dc-4d9d-8b8c-ceac2ecd0d0f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:45.712996874Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 274 len 4096 data: 13 03/30 Write block 414 len 4096 data: 10 04/30 Read block 523 len 28672 05/30 Flush 06/30 Write block 280 len 28672 data: 6 6 9 8 6 7 6 07/30 Write block 209 len 32768 data: 10 8 8 9 8 7 9 9 08/30 Write block 265 len 28672 data: 9 9 9 9 11 10 11 09/30 Read block 294 len 4096 10/30 Read block 492 len 4096 11/30 Read block 517 len 24576 12/30 Write block 559 len 40960 data: 18 14 13 14 12 9 6 5 6 9 13/30 Read block 7 len 32768 14/30 Read block 519 len 4096 15/30 Read block 374 len 16384 16/30 Read block 15 len 16384 17/30 Write block 110 len 20480 data: 5 7 9 11 10 18/30 Read block 104 len 32768 19/30 Read block 60 len 40960 20/30 Write block 101 len 36864 data: 12 11 12 10 8 7 7 5 5 21/30 Write block 283 len 24576 data: 9 7 8 7 7 7 22/30 Write block 219 len 16384 data: 5 6 6 9 23/30 Flush 24/30 Flush 25/30 Write block 131 len 20480 data: 8 7 6 7 9 26/30 Write block 204 len 36864 data: 12 13 11 11 11 11 9 9 10 27/30 Write block 341 len 12288 data: 13 14 12 28/30 Read block 366 len 12288 29/30 Read block 383 len 16384 30/30 Read block 39 len 36864 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 29 28 1 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:45.732671485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1716} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 207 207 207 F F F 1 020-039 124 124 124 211 211 211 F F F 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 126 126 124 217 217 210 F F F <--- 6 120-139 124 124 124 210 210 210 T T F <--- 7 140-159 122 122 122 207 207 207 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 124 124 124 210 210 210 F F F 10 200-219 126 126 116 217 217 200 T T F <--- 11 220-239 126 126 118 217 217 203 F F F <--- 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 122 217 217 207 F F T <--- 14 280-299 126 126 124 217 217 210 F F F <--- 15 300-319 122 122 122 207 207 207 F F F 16 320-339 122 122 122 207 207 207 F F F 17 340-359 120 120 120 205 205 205 T T F <--- 18 360-379 125 125 125 214 214 214 F F F 19 380-399 125 125 125 214 214 214 F F F 20 400-419 126 126 122 216 216 207 F F F <--- 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 122 122 122 207 207 207 F F F 27 540-559 126 126 124 217 217 213 F F F <--- 28 560-579 126 126 124 217 217 210 F F F <--- 29 580-599 124 124 124 213 213 213 F F F Max gen: 126, Max flush: 217 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 207 207 207 F F F 1 020-039 124 124 124 211 211 211 F F F 2 040-059 120 120 120 205 205 205 F F F 3 060-079 120 120 120 205 205 205 F F F 4 080-099 120 120 120 205 205 205 F F F 5 100-119 126 126 126 217 217 217 F F F 6 120-139 127 127 127 218 218 218 F F F 7 140-159 122 122 122 207 207 207 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 124 124 124 210 210 210 F F F 10 200-219 127 127 127 218 218 218 F F F 11 220-239 126 126 126 217 217 217 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 126 126 126 217 217 217 F F F 15 300-319 122 122 122 207 207 207 F F F 16 320-339 122 122 122 207 207 207 F F F 17 340-359 127 127 127 218 218 218 F F F 18 360-379 125 125 125 214 214 214 F F F 19 380-399 125 125 125 214 214 214 F F F 20 400-419 126 126 126 216 216 216 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 122 122 122 207 207 207 F F F 27 540-559 126 126 126 217 217 217 F F F 28 560-579 126 126 126 217 217 217 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 127, Max flush: 218 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.170201116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.216480873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.217826352Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.217892629Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.217914088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.217941437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} {"msg":"Crucible 0d431f82-65ca-4410-bdae-6355c8e687eb has session id: 25fb5605-06d9-47b5-b825-4259af6510c9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.218625531Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Upstairs opts: Upstairs UUID: 0d431f82-65ca-4410-bdae-6355c8e687eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.218691579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Crucible stats registered with UUID: 0d431f82-65ca-4410-bdae-6355c8e687eb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.218723057Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:50.218749336Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.218982628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219055875Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219127292Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219359923Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219391192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} {"msg":"0d431f82-65ca-4410-bdae-6355c8e687eb active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219470819Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219491428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"0","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219515847Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"1","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219545656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"2","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219576935Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219611224Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219644752Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54282)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.21970304Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52083)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219740119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:32833)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.219779377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.221253131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"1","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.221305189Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"0","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.221997133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.222029251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.222281782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"2","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[0]R flush_numbers[0..12]: [207, 211, 205, 205, 205, 217, 218, 207, 199, 210, 218, 217]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.222918728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[0]R generation[0..12]: [122, 124, 120, 120, 120, 126, 127, 122, 115, 124, 127, 126]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.222944317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.222964076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[1]R flush_numbers[0..12]: [207, 211, 205, 205, 205, 217, 218, 207, 199, 210, 218, 217]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.222983185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[1]R generation[0..12]: [122, 124, 120, 120, 120, 126, 127, 122, 115, 124, 127, 126]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223002094Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223021854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[2]R flush_numbers[0..12]: [207, 211, 205, 205, 205, 217, 218, 207, 199, 210, 218, 217]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223040803Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[2]R generation[0..12]: [122, 124, 120, 120, 120, 126, 127, 122, 115, 124, 127, 126]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223059772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223079531Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223095851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.22311303Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:02:50.223136359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223160568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"":"downstairs","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223177858Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223193307Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"0","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223210037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"1","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223227006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"client":"2","":"downstairs","session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223243925Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223259935Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"0d431f82-65ca-4410-bdae-6355c8e687eb is now active with session: 25fb5605-06d9-47b5-b825-4259af6510c9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223278024Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223293123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724,"session_id":"25fb5605-06d9-47b5-b825-4259af6510c9"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223309103Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} {"msg":"Activated sub_volume 0d431f82-65ca-4410-bdae-6355c8e687eb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:50.223322932Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 67 len 32768 data: 10 8 8 7 6 9 11 9 02/30 Read block 105 len 4096 03/30 Read block 3 len 16384 04/30 Read block 210 len 12288 05/30 Read block 226 len 40960 06/30 Read block 45 len 36864 07/30 Read block 85 len 28672 08/30 Read block 379 len 28672 09/30 Write block 83 len 40960 data: 10 10 11 13 11 11 11 7 6 8 10/30 Read block 121 len 32768 11/30 Write block 17 len 32768 data: 13 10 11 14 17 14 13 16 12/30 Read block 2 len 8192 13/30 Flush 14/30 Write block 326 len 40960 data: 6 5 6 6 7 7 7 8 7 8 15/30 Write block 47 len 8192 data: 12 13 16/30 Write block 157 len 12288 data: 9 9 11 17/30 Read block 39 len 20480 18/30 Write block 542 len 36864 data: 8 9 10 7 8 7 9 8 9 19/30 Flush 20/30 Write block 528 len 40960 data: 6 8 9 9 9 9 11 11 12 12 21/30 Read block 341 len 36864 22/30 Read block 89 len 20480 23/30 Read block 371 len 12288 24/30 Write block 68 len 4096 data: 9 25/30 Read block 516 len 32768 26/30 Write block 372 len 12288 data: 12 11 9 27/30 Read block 95 len 8192 28/30 Write block 287 len 12288 data: 8 8 7 29/30 Read block 318 len 36864 30/30 Write block 82 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:50.245898052Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1724} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 122 219 219 207 F F F <--- 1 020-039 128 128 124 219 219 211 F F F <--- 2 040-059 128 128 120 220 220 205 F F F <--- 3 060-079 128 128 120 219 219 205 T T T <--- 4 080-099 128 128 120 219 219 205 T T F <--- 5 100-119 126 126 126 217 217 217 F F F 6 120-139 127 127 127 218 218 218 F F F 7 140-159 128 128 122 220 220 207 F F F <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 124 124 124 210 210 210 F F F 10 200-219 127 127 127 218 218 218 F F F 11 220-239 126 126 126 217 217 217 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 126 126 126 217 217 217 T T F <--- 15 300-319 122 122 122 207 207 207 F F F 16 320-339 128 128 122 220 220 207 F F F <--- 17 340-359 127 127 127 218 218 218 F F F 18 360-379 125 125 125 214 214 214 T T F <--- 19 380-399 125 125 125 214 214 214 F F F 20 400-419 126 126 126 216 216 216 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 122 122 122 207 207 207 T T F <--- 27 540-559 128 128 126 220 220 217 F F F <--- 28 560-579 126 126 126 217 217 217 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 128, Max flush: 220 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 128 128 128 219 219 219 F F F 2 040-059 128 128 128 220 220 220 F F F 3 060-079 129 129 129 221 221 221 F F F 4 080-099 129 129 129 221 221 221 F F F 5 100-119 126 126 126 217 217 217 F F F 6 120-139 127 127 127 218 218 218 F F F 7 140-159 128 128 128 220 220 220 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 124 124 124 210 210 210 F F F 10 200-219 127 127 127 218 218 218 F F F 11 220-239 126 126 126 217 217 217 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 122 122 122 207 207 207 F F F 16 320-339 128 128 128 220 220 220 F F F 17 340-359 127 127 127 218 218 218 F F F 18 360-379 129 129 129 221 221 221 F F F 19 380-399 125 125 125 214 214 214 F F F 20 400-419 126 126 126 216 216 216 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 129 129 129 221 221 221 F F F 27 540-559 128 128 128 220 220 220 F F F 28 560-579 126 126 126 217 217 217 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 129, Max flush: 221 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.676080414Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.723111192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.723904382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.723966119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.723991658Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.724020857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} {"msg":"Crucible 20df5aec-8dc7-429e-8d44-178cb38e670a has session id: 211f467e-091e-4380-829c-60285d39ad2a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.724614745Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Upstairs opts: Upstairs UUID: 20df5aec-8dc7-429e-8d44-178cb38e670a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.724671723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Crucible stats registered with UUID: 20df5aec-8dc7-429e-8d44-178cb38e670a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.724692812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:54.724710021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.724908994Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.724958722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.72499022Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725107416Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725137155Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} {"msg":"20df5aec-8dc7-429e-8d44-178cb38e670a active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725211982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725234521Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"0","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.72526052Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"1","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725291729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725329427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725364036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725401095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:54.725442133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:54.725480252Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53942)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725514591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50104)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725545529Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:02:54.725585578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725618907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.725653445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.726782122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"1","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.726870489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"0","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.72736897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:02:54.727427068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.726344449Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43429)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.726418356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728056534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[0]R flush_numbers[0..12]: [219, 219, 220, 221, 221, 217, 218, 220, 199, 210, 218, 217]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728710179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[0]R generation[0..12]: [128, 128, 128, 129, 129, 126, 127, 128, 115, 124, 127, 126]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728749767Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728776446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[1]R flush_numbers[0..12]: [219, 219, 220, 221, 221, 217, 218, 220, 199, 210, 218, 217]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728800595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[1]R generation[0..12]: [128, 128, 128, 129, 129, 126, 127, 128, 115, 124, 127, 126]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728824475Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728849463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[2]R flush_numbers[0..12]: [219, 219, 220, 221, 221, 217, 218, 220, 199, 210, 218, 217]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728873263Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[2]R generation[0..12]: [128, 128, 128, 129, 129, 126, 127, 128, 115, 124, 127, 126]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728896852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:03:04.72892864Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728966759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.728994638Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729039576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729057866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"":"downstairs","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729075445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729090994Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"0","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729108614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"1","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729125923Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"client":"2","":"downstairs","session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729143642Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729159772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"20df5aec-8dc7-429e-8d44-178cb38e670a is now active with session: 211f467e-091e-4380-829c-60285d39ad2a","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-02-15T00:03:04.729177901Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.72919533Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732,"session_id":"211f467e-091e-4380-829c-60285d39ad2a"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.72921187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} {"msg":"Activated sub_volume 20df5aec-8dc7-429e-8d44-178cb38e670a","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:04.729226129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 360 len 36864 data: 6 8 8 9 8 10 9 9 13 02/30 Write block 137 len 40960 data: 13 14 12 11 12 14 12 10 8 9 03/30 Read block 301 len 24576 04/30 Write block 222 len 28672 data: 10 11 10 7 5 6 8 05/30 Write block 207 len 16384 data: 12 12 12 10 06/30 Write block 181 len 8192 data: 3 4 07/30 Write block 35 len 36864 data: 5 4 4 8 8 10 11 10 11 08/30 Read block 590 len 4096 09/30 Read block 371 len 20480 10/30 Flush 11/30 Write block 471 len 8192 data: 8 8 12/30 Write block 60 len 36864 data: 14 11 12 9 9 9 10 11 10 13/30 Read block 492 len 40960 14/30 Read block 239 len 24576 15/30 Read block 312 len 32768 16/30 Read block 9 len 16384 17/30 Write block 389 len 40960 data: 8 9 8 9 12 11 12 13 12 11 18/30 Read block 563 len 28672 19/30 Read block 317 len 32768 20/30 Read block 48 len 20480 21/30 Read block 46 len 8192 22/30 Read block 334 len 20480 23/30 Write block 547 len 8192 data: 8 10 24/30 Write block 37 len 40960 data: 5 9 9 11 12 11 12 10 8 9 25/30 Read block 517 len 36864 26/30 Read block 6 len 4096 27/30 Read block 584 len 4096 28/30 Read block 494 len 28672 29/30 Write block 128 len 4096 data: 11 30/30 Read block 219 len 4096 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 6 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:04.749843234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1732} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 130 130 128 222 222 219 T T F <--- 2 040-059 130 130 128 222 222 220 T T F <--- 3 060-079 129 129 129 221 221 221 T T F <--- 4 080-099 129 129 129 221 221 221 F F F 5 100-119 126 126 126 217 217 217 F F F 6 120-139 130 130 127 222 222 218 T T T <--- 7 140-159 130 130 128 222 222 220 F F T <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 130 130 124 222 222 210 F F F <--- 10 200-219 130 130 127 222 222 218 F F F <--- 11 220-239 130 130 126 222 222 217 F F F <--- 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 122 122 122 207 207 207 F F F 16 320-339 128 128 128 220 220 220 F F F 17 340-359 127 127 127 218 218 218 F F F 18 360-379 130 130 129 222 222 221 F F T <--- 19 380-399 125 125 125 214 214 214 T T F <--- 20 400-419 126 126 126 216 216 216 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 120 120 120 205 205 205 T T F <--- 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 129 129 129 221 221 221 F F F 27 540-559 128 128 128 220 220 220 T T F <--- 28 560-579 126 126 126 217 217 217 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 130, Max flush: 222 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 131 131 131 223 223 223 F F F 3 060-079 131 131 131 223 223 223 F F F 4 080-099 129 129 129 221 221 221 F F F 5 100-119 126 126 126 217 217 217 F F F 6 120-139 131 131 131 223 223 223 F F F 7 140-159 130 130 130 222 222 222 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 130 130 130 222 222 222 F F F 10 200-219 130 130 130 222 222 222 F F F 11 220-239 130 130 130 222 222 222 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 122 122 122 207 207 207 F F F 16 320-339 128 128 128 220 220 220 F F F 17 340-359 127 127 127 218 218 218 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 131 131 131 223 223 223 F F F 20 400-419 126 126 126 216 216 216 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 F F F 23 460-479 131 131 131 223 223 223 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 129 129 129 221 221 221 F F F 27 540-559 131 131 131 223 223 223 F F F 28 560-579 126 126 126 217 217 217 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 131, Max flush: 223 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.164944589Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.208986701Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.209740283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.209791071Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.20981284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.209841679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} {"msg":"Crucible 8f5dc4aa-78ca-42ba-a636-d952bc68f7ed has session id: 5f34902d-0c3e-409c-917d-860dfc9cfadd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.210468275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Upstairs opts: Upstairs UUID: 8f5dc4aa-78ca-42ba-a636-d952bc68f7ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.210517153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Crucible stats registered with UUID: 8f5dc4aa-78ca-42ba-a636-d952bc68f7ed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.210536893Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:09.210552992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.210783313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.210825362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.210844481Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211040453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211062813Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} {"msg":"8f5dc4aa-78ca-42ba-a636-d952bc68f7ed active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211147759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211168369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"0","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211185578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"1","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211202397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"2","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211231776Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211322453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211359931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45860)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.21139097Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61616)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211423109Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"ds_connection connected from Ok(127.0.0.1:48121)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.211457067Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.212628943Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"0","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.212770357Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"1","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.213289998Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.213398033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.214377496Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"2","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 223, 223, 221, 217, 223, 222, 199, 222, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.214908576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[0]R generation[0..12]: [128, 131, 131, 131, 129, 126, 131, 130, 115, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.214934795Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.214954994Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 223, 223, 221, 217, 223, 222, 199, 222, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.214977403Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[1]R generation[0..12]: [128, 131, 131, 131, 129, 126, 131, 130, 115, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.214996753Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215016532Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 223, 223, 221, 217, 223, 222, 199, 222, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215035451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[2]R generation[0..12]: [128, 131, 131, 131, 129, 126, 131, 130, 115, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215054201Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.21507371Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215099269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215138857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215161736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215184105Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"":"downstairs","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215212864Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215238943Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"0","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215269222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"client":"1","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215300411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1740,"client":"2","":"downstairs","session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.21533222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215354009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"8f5dc4aa-78ca-42ba-a636-d952bc68f7ed is now active with session: 5f34902d-0c3e-409c-917d-860dfc9cfadd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215372718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215388088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740,"session_id":"5f34902d-0c3e-409c-917d-860dfc9cfadd"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215404197Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} {"msg":"Activated sub_volume 8f5dc4aa-78ca-42ba-a636-d952bc68f7ed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:09.215417967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 32768 data: 6 7 7 6 7 7 8 8 02/30 Read block 30 len 20480 03/30 Write block 355 len 8192 data: 7 7 04/30 Read block 327 len 32768 05/30 Flush 06/30 Write block 64 len 4096 data: 10 07/30 Read block 122 len 40960 08/30 Read block 574 len 8192 09/30 Write block 50 len 32768 data: 11 12 12 14 15 10 9 8 10/30 Read block 24 len 28672 11/30 Write block 84 len 40960 data: 11 12 14 12 12 12 8 7 9 8 12/30 Read block 564 len 40960 13/30 Write block 569 len 24576 data: 11 13 15 14 16 16 14/30 Read block 542 len 32768 15/30 Write block 148 len 36864 data: 12 11 11 10 9 11 9 11 10 16/30 Write block 126 len 32768 data: 8 9 12 8 9 9 8 7 17/30 Write block 187 len 8192 data: 9 11 18/30 Write block 395 len 32768 data: 13 14 13 12 10 10 8 8 19/30 Read block 422 len 32768 20/30 Read block 324 len 36864 21/30 Read block 177 len 4096 22/30 Flush 23/30 Flush 24/30 Write block 302 len 16384 data: 18 16 13 15 25/30 Read block 490 len 16384 26/30 Read block 48 len 4096 27/30 Write block 465 len 8192 data: 7 5 28/30 Read block 402 len 4096 29/30 Write block 456 len 32768 data: 8 7 6 6 6 7 9 8 30/30 Read block 593 len 28672 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 2 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:09.235498772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1740} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 131 225 225 223 F F F <--- 3 060-079 132 132 131 225 225 223 F F F <--- 4 080-099 132 132 129 225 225 221 F F F <--- 5 100-119 126 126 126 217 217 217 F F F 6 120-139 132 132 131 225 225 223 F F F <--- 7 140-159 132 132 130 225 225 222 F F F <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 132 132 130 225 225 222 F F F <--- 10 200-219 130 130 130 222 222 222 F F F 11 220-239 130 130 130 222 222 222 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 122 122 122 207 207 207 T T F <--- 16 320-339 132 132 128 224 224 220 F F T <--- 17 340-359 132 132 127 224 224 218 F F F <--- 18 360-379 130 130 130 222 222 222 F F F 19 380-399 132 132 131 225 225 223 F F F <--- 20 400-419 132 132 126 225 225 216 F F F <--- 21 420-439 124 124 124 211 211 211 F F F 22 440-459 120 120 120 205 205 205 T T F <--- 23 460-479 131 131 131 223 223 223 T T F <--- 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 129 129 129 221 221 221 F F F 27 540-559 131 131 131 223 223 223 F F F 28 560-579 132 132 126 225 225 217 F F F <--- 29 580-599 124 124 124 213 213 213 F F F Max gen: 132, Max flush: 225 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 132 132 132 225 225 225 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 126 126 126 217 217 217 F F F 6 120-139 132 132 132 225 225 225 F F F 7 140-159 132 132 132 225 225 225 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 130 130 130 222 222 222 F F F 11 220-239 130 130 130 222 222 222 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 133 133 133 226 226 226 F F F 16 320-339 132 132 132 224 224 224 F F F 17 340-359 132 132 132 224 224 224 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 132 132 132 225 225 225 F F F 20 400-419 132 132 132 225 225 225 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 133 133 133 226 226 226 F F F 23 460-479 133 133 133 226 226 226 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 129 129 129 221 221 221 F F F 27 540-559 131 131 131 223 223 223 F F F 28 560-579 132 132 132 225 225 225 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 133, Max flush: 226 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.650393385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.696754559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.697572048Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.697638286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.697662994Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.697694953Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} {"msg":"Crucible 350655b0-06f8-47f3-acec-80350f7c0565 has session id: cf26db7a-e547-4039-8248-18b531dbe6eb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698413086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Upstairs opts: Upstairs UUID: 350655b0-06f8-47f3-acec-80350f7c0565, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698477143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Crucible stats registered with UUID: 350655b0-06f8-47f3-acec-80350f7c0565","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698503973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:13.698524502Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698684566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698730194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698750753Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698893828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698918677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} {"msg":"350655b0-06f8-47f3-acec-80350f7c0565 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.698982354Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699003194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"0","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699020523Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"1","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699037232Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"2","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699063251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.69909493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699163657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41234)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699253224Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:61509)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699304702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57989)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.699452836Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.70066182Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"1","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.700778036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"2","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.700805235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"0","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701245838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701320065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 225, 225, 217, 225, 225, 199, 225, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701428011Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 132, 132, 126, 132, 132, 115, 132, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.70145212Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.70147224Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 225, 225, 217, 225, 225, 199, 225, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701491319Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 132, 132, 126, 132, 132, 115, 132, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701510308Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701530077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 225, 225, 217, 225, 225, 199, 225, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701549126Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 132, 132, 126, 132, 132, 115, 132, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701567926Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701587755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701607904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Generation requested: 134 >= found:134"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701631483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701649003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701665152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"":"downstairs","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701681961Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701697231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"0","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.70171406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"1","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701730559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"client":"2","":"downstairs","session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701747819Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701763818Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"350655b0-06f8-47f3-acec-80350f7c0565 is now active with session: cf26db7a-e547-4039-8248-18b531dbe6eb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701781868Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701801437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748,"session_id":"cf26db7a-e547-4039-8248-18b531dbe6eb"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701817546Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} {"msg":"Activated sub_volume 350655b0-06f8-47f3-acec-80350f7c0565","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:13.701831446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 512 len 36864 data: 4 7 7 9 12 13 14 12 12 02/30 Read block 359 len 24576 03/30 Flush 04/30 Write block 152 len 32768 data: 10 12 10 12 11 10 10 12 05/30 Read block 225 len 16384 06/30 Write block 60 len 28672 data: 15 12 13 10 11 10 11 07/30 Read block 217 len 8192 08/30 Write block 68 len 12288 data: 11 9 8 09/30 Read block 498 len 20480 10/30 Read block 345 len 36864 11/30 Read block 65 len 20480 12/30 Read block 177 len 4096 13/30 Read block 302 len 32768 14/30 Write block 341 len 4096 data: 14 15/30 Write block 202 len 40960 data: 12 10 13 14 12 13 13 13 11 10 16/30 Write block 101 len 36864 data: 13 12 13 11 9 8 8 6 6 17/30 Write block 316 len 32768 data: 8 10 11 14 13 14 12 10 18/30 Read block 224 len 8192 19/30 Write block 340 len 16384 data: 17 15 15 13 20/30 Read block 579 len 32768 21/30 Write block 503 len 24576 data: 15 13 13 11 7 7 22/30 Write block 152 len 12288 data: 11 13 11 23/30 Read block 516 len 4096 24/30 Write block 114 len 20480 data: 11 11 13 16 15 25/30 Write block 398 len 36864 data: 13 11 11 9 9 13 12 13 14 26/30 Read block 464 len 36864 27/30 Write block 382 len 12288 data: 6 5 5 28/30 Read block 192 len 4096 29/30 Read block 350 len 36864 30/30 Read block 252 len 20480 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:13.721876112Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1748} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 132 132 132 225 225 225 T T F <--- 4 080-099 132 132 132 225 225 225 F F F 5 100-119 126 126 126 217 217 217 T T F <--- 6 120-139 132 132 132 225 225 225 F F F 7 140-159 132 132 132 225 225 225 T T F <--- 8 160-179 115 115 115 199 199 199 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 130 130 130 222 222 222 T T F <--- 11 220-239 130 130 130 222 222 222 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 133 133 133 226 226 226 T T F <--- 16 320-339 132 132 132 224 224 224 T T F <--- 17 340-359 132 132 132 224 224 224 T T F <--- 18 360-379 130 130 130 222 222 222 F F F 19 380-399 132 132 132 225 225 225 T T F <--- 20 400-419 132 132 132 225 225 225 T T F <--- 21 420-439 124 124 124 211 211 211 F F F 22 440-459 133 133 133 226 226 226 F F F 23 460-479 133 133 133 226 226 226 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 134 134 124 227 227 208 T T F <--- 26 520-539 134 134 129 227 227 221 F F F <--- 27 540-559 131 131 131 223 223 223 F F F 28 560-579 132 132 132 225 225 225 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 134, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 135 135 135 228 228 228 F F F 4 080-099 132 132 132 225 225 225 F F F 5 100-119 135 135 135 228 228 228 F F F 6 120-139 132 132 132 225 225 225 F F F 7 140-159 135 135 135 228 228 228 F F F 8 160-179 115 115 115 199 199 199 F F F 9 180-199 132 132 132 225 225 225 F F F 10 200-219 135 135 135 228 228 228 F F F 11 220-239 130 130 130 222 222 222 F F F 12 240-259 125 125 125 214 214 214 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 135 135 135 228 228 228 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 135 135 135 228 228 228 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 135 135 135 228 228 228 F F F 20 400-419 135 135 135 228 228 228 F F F 21 420-439 124 124 124 211 211 211 F F F 22 440-459 133 133 133 226 226 226 F F F 23 460-479 133 133 133 226 226 226 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 135 135 135 228 228 228 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 131 131 131 223 223 223 F F F 28 560-579 132 132 132 225 225 225 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 135, Max flush: 228 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.135754734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:18.179052615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.17943163Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.210268366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.21095357Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211017897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211039666Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211068835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} {"msg":"Crucible b701ef01-ea14-495d-bc07-ef2e9a6ba2c5 has session id: fd7e6021-aa8b-4e58-8a9d-55899c4338b6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211647533Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Upstairs opts: Upstairs UUID: b701ef01-ea14-495d-bc07-ef2e9a6ba2c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211715991Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Crucible stats registered with UUID: b701ef01-ea14-495d-bc07-ef2e9a6ba2c5","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211743839Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:18.211761749Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211919263Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.211973201Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212026119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212200082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212230801Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} {"msg":"b701ef01-ea14-495d-bc07-ef2e9a6ba2c5 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212277689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212297738Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"0","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212317708Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"1","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212335887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"2","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212353776Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212379015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212409294Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43545)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212433233Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35139)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212560828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61771)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.212661735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214003424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"1","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214048122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"2","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214070851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"0","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214705847Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214736945Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 228, 225, 228, 225, 228, 199, 225, 228, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214763735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 135, 132, 135, 132, 135, 115, 132, 135, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214783694Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214803313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 228, 225, 228, 225, 228, 199, 225, 228, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214822312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 135, 132, 135, 132, 135, 115, 132, 135, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214841032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214860571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 228, 225, 228, 225, 228, 199, 225, 228, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.21487957Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 135, 132, 135, 132, 135, 115, 132, 135, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214898339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214917989Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214934488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214951347Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214967457Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.214983626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"":"downstairs","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215000486Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"Wait for a query_work_queue command to finish before sending IO fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215027185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"0","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215044274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"1","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215060873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"client":"2","":"downstairs","session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215078032Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215094152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"b701ef01-ea14-495d-bc07-ef2e9a6ba2c5 is now active with session: fd7e6021-aa8b-4e58-8a9d-55899c4338b6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215112051Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.21513816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756,"session_id":"fd7e6021-aa8b-4e58-8a9d-55899c4338b6"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.21515532Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} {"msg":"Activated sub_volume b701ef01-ea14-495d-bc07-ef2e9a6ba2c5","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:18.215169219Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 24576 data: 11 10 8 9 8 5 02/30 Write block 412 len 16384 data: 11 10 11 7 03/30 Write block 192 len 32768 data: 6 6 7 8 8 11 10 12 04/30 Write block 422 len 28672 data: 13 11 15 13 12 14 12 05/30 Read block 147 len 8192 06/30 Write block 401 len 36864 data: 10 10 14 13 14 15 13 13 12 07/30 Write block 493 len 16384 data: 12 10 10 11 08/30 Read block 62 len 16384 09/30 Read block 270 len 24576 10/30 Read block 426 len 40960 11/30 Write block 481 len 36864 data: 6 7 11 13 14 13 16 16 15 12/30 Write block 159 len 20480 data: 13 11 12 11 11 13/30 Write block 94 len 36864 data: 8 10 9 10 9 9 13 14 13 14/30 Read block 387 len 8192 15/30 Read block 342 len 40960 16/30 Write block 171 len 36864 data: 10 12 12 11 12 11 9 10 9 17/30 Write block 249 len 20480 data: 7 10 10 11 12 18/30 Write block 559 len 28672 data: 19 15 14 15 13 10 7 19/30 Read block 357 len 36864 20/30 Read block 161 len 16384 21/30 Write block 563 len 36864 data: 14 11 8 6 7 10 12 14 16 22/30 Write block 479 len 20480 data: 7 6 7 8 12 23/30 Read block 354 len 4096 24/30 Read block 426 len 16384 25/30 Write block 437 len 8192 data: 13 13 26/30 Flush 27/30 Write block 481 len 40960 data: 8 9 13 14 15 14 17 17 16 17 28/30 Write block 170 len 20480 data: 11 11 13 13 12 29/30 Write block 118 len 32768 data: 16 13 14 14 14 14 12 10 30/30 Write block 544 len 32768 data: 11 8 9 9 11 9 10 10 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:18.235191157Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1756} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 135 135 135 228 228 228 F F F 4 080-099 132 136 136 225 229 229 F F F <--- 5 100-119 135 136 136 228 229 229 F T T <--- 6 120-139 132 132 132 225 225 225 F T T <--- 7 140-159 135 136 136 228 229 229 F F F <--- 8 160-179 115 136 136 199 229 229 T T T <--- 9 180-199 132 136 136 225 229 229 T F F <--- 10 200-219 135 135 135 228 228 228 F F F 11 220-239 130 130 130 222 222 222 F F F 12 240-259 125 136 136 214 229 229 F F F <--- 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 135 135 135 228 228 228 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 135 135 135 228 228 228 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 135 135 135 228 228 228 F F F 20 400-419 135 136 136 228 229 229 F F F <--- 21 420-439 124 136 136 211 229 229 F F F <--- 22 440-459 133 133 133 226 226 226 F F F 23 460-479 133 136 136 226 229 229 F F F <--- 24 480-499 124 136 136 208 229 229 F T T <--- 25 500-519 135 135 135 228 228 228 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 131 136 136 223 229 229 F T T <--- 28 560-579 132 136 136 225 229 229 F F F <--- 29 580-599 124 124 124 213 213 213 F F F Max gen: 136, Max flush: 229 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 135 135 135 228 228 228 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 137 137 137 230 230 230 F F F 6 120-139 137 137 137 230 230 230 F F F 7 140-159 136 136 136 229 229 229 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 136 136 136 229 229 229 F F F 10 200-219 135 135 135 228 228 228 F F F 11 220-239 130 130 130 222 222 222 F F F 12 240-259 136 136 136 229 229 229 F F F 13 260-279 126 126 126 217 217 217 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 135 135 135 228 228 228 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 135 135 135 228 228 228 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 135 135 135 228 228 228 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 136 136 136 229 229 229 F F F 22 440-459 133 133 133 226 226 226 F F F 23 460-479 136 136 136 229 229 229 F F F 24 480-499 137 137 137 230 230 230 F F F 25 500-519 135 135 135 228 228 228 F F F 26 520-539 134 134 134 227 227 227 F F F 27 540-559 137 137 137 230 230 230 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 137, Max flush: 230 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.718054891Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.767536546Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.768408333Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.768458791Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.76847448Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.768501669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} {"msg":"Crucible 4cab9775-22d2-41cf-91ca-0e814de10bfd has session id: b6649836-4cbd-4061-bce5-d950cd511e9c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769103376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Upstairs opts: Upstairs UUID: 4cab9775-22d2-41cf-91ca-0e814de10bfd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769235651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Crucible stats registered with UUID: 4cab9775-22d2-41cf-91ca-0e814de10bfd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.76927492Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:22.769304499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.76952419Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769581628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769601367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.7697985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769819999Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} {"msg":"4cab9775-22d2-41cf-91ca-0e814de10bfd active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769888146Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769907776Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"0","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769925215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"1","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769941904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"2","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769958424Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.769975603Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.770032771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57329)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.770161026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60489)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.770210214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36791)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.770372068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.771694857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"0","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.771731026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"2","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.771753025Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"1","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772391651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.7724222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 228, 229, 230, 230, 229, 230, 229, 228, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772452099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 135, 136, 137, 137, 136, 137, 136, 135, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772472348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772492077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 228, 229, 230, 230, 229, 230, 229, 228, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772511706Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 135, 136, 137, 137, 136, 137, 136, 135, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772530596Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772553815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 228, 229, 230, 230, 229, 230, 229, 228, 222]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772573054Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 135, 136, 137, 137, 136, 137, 136, 135, 130]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772591814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772611163Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772627372Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772644021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772660141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:03:22.77267636Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"":"downstairs","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772699209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772714399Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"0","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772730908Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"1","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772747377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"client":"2","":"downstairs","session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772764247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772780216Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"4cab9775-22d2-41cf-91ca-0e814de10bfd is now active with session: b6649836-4cbd-4061-bce5-d950cd511e9c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772798106Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772815365Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764,"session_id":"b6649836-4cbd-4061-bce5-d950cd511e9c"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772831434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} {"msg":"Activated sub_volume 4cab9775-22d2-41cf-91ca-0e814de10bfd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:22.772845164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 8192 data: 16 16 02/30 Read block 486 len 8192 03/30 Read block 371 len 16384 04/30 Read block 468 len 36864 05/30 Read block 49 len 4096 06/30 Read block 217 len 24576 07/30 Read block 476 len 36864 08/30 Read block 21 len 24576 09/30 Read block 393 len 16384 10/30 Write block 348 len 4096 data: 12 11/30 Read block 517 len 40960 12/30 Read block 140 len 16384 13/30 Write block 393 len 12288 data: 13 12 14 14/30 Write block 235 len 36864 data: 11 11 11 8 9 7 6 8 8 15/30 Read block 414 len 4096 16/30 Write block 269 len 24576 data: 12 11 12 12 11 14 17/30 Write block 527 len 4096 data: 7 18/30 Read block 537 len 20480 19/30 Read block 584 len 24576 20/30 Write block 517 len 4096 data: 14 21/30 Read block 588 len 32768 22/30 Flush 23/30 Flush 24/30 Write block 191 len 8192 data: 9 7 25/30 Flush 26/30 Write block 150 len 20480 data: 12 11 12 14 12 27/30 Write block 151 len 4096 data: 12 28/30 Write block 132 len 20480 data: 9 8 8 10 10 29/30 Read block 308 len 4096 30/30 Read block 442 len 40960 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 0 22 Done 30 8 30 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:22.791452215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1764} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 135 135 135 228 228 228 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 137 137 137 230 230 230 F F F 6 120-139 137 137 137 230 230 230 T F T <--- 7 140-159 136 136 136 229 229 229 T F T <--- 8 160-179 137 137 137 230 230 230 F F F 9 180-199 138 136 138 233 229 233 F F F <--- 10 200-219 135 135 135 228 228 228 F F F 11 220-239 138 130 138 231 222 231 F F F <--- 12 240-259 138 136 138 231 229 231 F F F <--- 13 260-279 138 126 138 231 217 231 F F F <--- 14 280-299 129 129 129 221 221 221 F F F 15 300-319 135 135 135 228 228 228 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 135 138 231 228 231 F F F <--- 18 360-379 130 130 130 222 222 222 F F F 19 380-399 138 135 138 231 228 231 F F F <--- 20 400-419 136 136 136 229 229 229 F F F 21 420-439 136 136 136 229 229 229 F F F 22 440-459 133 133 133 226 226 226 F F F 23 460-479 136 136 136 229 229 229 F F F 24 480-499 137 137 137 230 230 230 F F F 25 500-519 138 135 138 231 228 231 F F F <--- 26 520-539 138 134 138 231 227 231 F F F <--- 27 540-559 138 137 138 231 230 231 F T F <--- 28 560-579 136 136 136 229 229 229 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 138, Max flush: 233 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 135 135 135 228 228 228 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 137 137 137 230 230 230 F F F 6 120-139 139 139 139 234 234 234 F F F 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 138 138 138 233 233 233 F F F 10 200-219 135 135 135 228 228 228 F F F 11 220-239 138 138 138 231 231 231 F F F 12 240-259 138 138 138 231 231 231 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 129 129 129 221 221 221 F F F 15 300-319 135 135 135 228 228 228 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 138 138 231 231 231 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 138 138 138 231 231 231 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 136 136 136 229 229 229 F F F 22 440-459 133 133 133 226 226 226 F F F 23 460-479 136 136 136 229 229 229 F F F 24 480-499 137 137 137 230 230 230 F F F 25 500-519 138 138 138 231 231 231 F F F 26 520-539 138 138 138 231 231 231 F F F 27 540-559 138 138 138 231 231 231 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 139, Max flush: 234 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.194186701Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.240483128Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.241291327Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.241340905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.241358224Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.241380453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} {"msg":"Crucible ed66f244-f560-492f-a375-0d19fe78ca30 has session id: 9d68f97a-04dc-48f3-bd58-5f0683ecf123","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242033379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Upstairs opts: Upstairs UUID: ed66f244-f560-492f-a375-0d19fe78ca30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242098866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Crucible stats registered with UUID: ed66f244-f560-492f-a375-0d19fe78ca30","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242123625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:27.242143744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242319618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242381825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242405404Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242553979Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242576988Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} {"msg":"ed66f244-f560-492f-a375-0d19fe78ca30 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242654065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242674744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"0","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242692214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"1","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242719082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"2","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242751751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.24278554Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242827658Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"ds_connection connected from Ok(127.0.0.1:58463)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242864167Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"ds_connection connected from Ok(127.0.0.1:36633)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242901666Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"ds_connection connected from Ok(127.0.0.1:42823)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.242940024Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.244434787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"2","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.244473376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"0","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.244769814Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"1","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245124841Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245206608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 228, 229, 230, 234, 234, 230, 233, 228, 231]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.2454156Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 135, 136, 137, 139, 139, 137, 138, 135, 138]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245441559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245461868Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 228, 229, 230, 234, 234, 230, 233, 228, 231]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245480707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 135, 136, 137, 139, 139, 137, 138, 135, 138]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245499876Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245519426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 228, 229, 230, 234, 234, 230, 233, 228, 231]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245538215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 135, 136, 137, 139, 139, 137, 138, 135, 138]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245557144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245576693Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245593053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245609772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Next flush: 235"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245632771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245653881Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"":"downstairs","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.24567108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245686499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"0","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245703179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"1","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245719778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"client":"2","":"downstairs","session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245736558Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245752537Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"ed66f244-f560-492f-a375-0d19fe78ca30 is now active with session: 9d68f97a-04dc-48f3-bd58-5f0683ecf123","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245770296Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245785436Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772,"session_id":"9d68f97a-04dc-48f3-bd58-5f0683ecf123"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245801445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} {"msg":"Activated sub_volume ed66f244-f560-492f-a375-0d19fe78ca30","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:27.245815204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 69 len 32768 data: 10 9 7 10 12 10 8 8 02/30 Write block 67 len 36864 data: 12 12 11 10 8 11 13 11 9 03/30 Flush 04/30 Write block 289 len 16384 data: 8 8 9 10 05/30 Flush 06/30 Write block 463 len 24576 data: 9 7 8 6 6 5 07/30 Write block 420 len 4096 data: 10 08/30 Read block 400 len 16384 09/30 Write block 516 len 32768 data: 13 15 15 13 13 10 12 11 10/30 Read block 35 len 32768 11/30 Write block 311 len 8192 data: 13 14 12/30 Read block 159 len 20480 13/30 Read block 5 len 20480 14/30 Read block 279 len 28672 15/30 Read block 431 len 24576 16/30 Read block 194 len 16384 17/30 Write block 438 len 24576 data: 14 15 14 12 9 9 18/30 Write block 191 len 24576 data: 10 8 7 8 9 9 19/30 Flush 20/30 Write block 488 len 16384 data: 18 17 18 16 21/30 Write block 427 len 24576 data: 15 13 10 9 11 9 22/30 Read block 506 len 16384 23/30 Write block 119 len 32768 data: 14 15 15 15 15 13 11 9 24/30 Flush 25/30 Write block 591 len 28672 data: 7 7 9 8 8 8 7 26/30 Read block 3 len 28672 27/30 Read block 533 len 24576 28/30 Read block 466 len 4096 29/30 Write block 549 len 32768 data: 10 11 11 10 11 12 11 13 30/30 Read block 249 len 4096 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:27.266701279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1772} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 140 135 140 235 228 235 F T F <--- 4 080-099 136 136 136 229 229 229 F F F 5 100-119 140 137 140 238 230 238 F F F <--- 6 120-139 140 139 140 238 234 238 F F F <--- 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 138 140 237 233 237 F F F <--- 10 200-219 135 135 135 228 228 228 F F F 11 220-239 138 138 138 231 231 231 F F F 12 240-259 138 138 138 231 231 231 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 140 129 140 236 221 236 F F F <--- 15 300-319 140 135 140 237 228 237 F F F <--- 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 138 138 231 231 231 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 138 138 138 231 231 231 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 140 136 140 238 229 238 F F F <--- 22 440-459 140 133 140 237 226 237 F F F <--- 23 460-479 140 136 140 237 229 237 F F F <--- 24 480-499 140 137 140 238 230 238 F F F <--- 25 500-519 140 138 140 237 231 237 F F F <--- 26 520-539 140 138 140 237 231 237 F F F <--- 27 540-559 138 138 138 231 231 231 T F T <--- 28 560-579 136 136 136 229 229 229 F F F 29 580-599 124 124 124 213 213 213 T F T <--- Max gen: 140, Max flush: 238 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 140 140 140 238 238 238 F F F 6 120-139 140 140 140 238 238 238 F F F 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 135 135 135 228 228 228 F F F 11 220-239 138 138 138 231 231 231 F F F 12 240-259 138 138 138 231 231 231 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 140 140 140 236 236 236 F F F 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 138 138 231 231 231 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 138 138 138 231 231 231 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 140 140 140 238 238 238 F F F 22 440-459 140 140 140 237 237 237 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 140 140 140 238 238 238 F F F 25 500-519 140 140 140 237 237 237 F F F 26 520-539 140 140 140 237 237 237 F F F 27 540-559 141 141 141 239 239 239 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 141 141 141 239 239 239 F F F Max gen: 141, Max flush: 239 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.683821118Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:31.727471995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.727890449Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.759048352Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.759804973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.759856051Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.759877441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.75990079Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} {"msg":"Crucible 2c1be2be-ef79-41d6-ad60-c19423f751e6 has session id: 2419f5d8-f545-49f7-a649-1d71057e6669","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.760347753Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Upstairs opts: Upstairs UUID: 2c1be2be-ef79-41d6-ad60-c19423f751e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.76041855Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Crucible stats registered with UUID: 2c1be2be-ef79-41d6-ad60-c19423f751e6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.760444629Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:31.760462539Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.760670771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.760722489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.760755187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.76093946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.760968899Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} {"msg":"2c1be2be-ef79-41d6-ad60-c19423f751e6 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761029407Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761067465Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"0","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761126413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"1","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761168281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"2","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.76120118Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761263858Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761304866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"ds_connection connected from Ok(127.0.0.1:50194)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761337755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"ds_connection connected from Ok(127.0.0.1:34248)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.7614835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"ds_connection connected from Ok(127.0.0.1:33459)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.761534468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.762635186Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"1","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.76278816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"0","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.762816389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"2","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763290321Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763414626Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 238, 238, 234, 230, 237, 228, 231]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763518122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 140, 136, 140, 140, 139, 137, 140, 135, 138]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763543611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.76356359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 238, 238, 234, 230, 237, 228, 231]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763587149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 140, 136, 140, 140, 139, 137, 140, 135, 138]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763606669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763626228Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 238, 238, 234, 230, 237, 228, 231]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763645117Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 140, 136, 140, 140, 139, 137, 140, 135, 138]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763663706Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763683446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763699575Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763716655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763732694Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763748723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"":"downstairs","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763765523Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763788672Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"0","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763805611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"1","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.7638222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"client":"2","":"downstairs","session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.76383915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763857349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"2c1be2be-ef79-41d6-ad60-c19423f751e6 is now active with session: 2419f5d8-f545-49f7-a649-1d71057e6669","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763875438Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763890498Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780,"session_id":"2419f5d8-f545-49f7-a649-1d71057e6669"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763906417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} {"msg":"Activated sub_volume 2c1be2be-ef79-41d6-ad60-c19423f751e6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:31.763920167Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 32768 data: 14 16 16 14 14 11 13 12 02/30 Read block 32 len 28672 03/30 Flush 04/30 Write block 230 len 16384 data: 10 11 11 9 05/30 Write block 390 len 4096 data: 10 06/30 Read block 315 len 36864 07/30 Flush 08/30 Write block 122 len 16384 data: 16 16 14 12 09/30 Read block 574 len 16384 10/30 Read block 348 len 20480 11/30 Read block 35 len 40960 12/30 Write block 285 len 8192 data: 9 8 13/30 Read block 465 len 32768 14/30 Read block 138 len 40960 15/30 Read block 260 len 36864 16/30 Read block 373 len 20480 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 389 len 36864 data: 9 11 9 10 14 13 15 15 14 21/30 Read block 73 len 32768 22/30 Read block 209 len 4096 23/30 Flush 24/30 Write block 207 len 24576 data: 14 14 14 12 11 11 25/30 Read block 351 len 4096 26/30 Read block 596 len 8192 27/30 Read block 335 len 20480 28/30 Read block 277 len 8192 29/30 Read block 382 len 8192 30/30 Read block 14 len 28672 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 2 29 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:31.783681003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1780} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 140 140 140 238 238 238 F F F 6 120-139 140 142 142 238 242 242 F F F <--- 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 135 135 135 228 228 228 F T T <--- 11 220-239 138 142 142 231 241 241 F F F <--- 12 240-259 138 138 138 231 231 231 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 140 142 142 236 242 242 F F F <--- 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 138 138 231 231 231 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 138 142 142 231 245 245 F F F <--- 20 400-419 136 136 136 229 229 229 F F F 21 420-439 140 140 140 238 238 238 F F F 22 440-459 140 140 140 237 237 237 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 140 140 140 238 238 238 F F F 25 500-519 140 142 142 237 240 240 T F F <--- 26 520-539 140 142 142 237 240 240 T F F <--- 27 540-559 141 141 141 239 239 239 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 141 141 141 239 239 239 F F F Max gen: 142, Max flush: 245 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 140 140 140 238 238 238 F F F 6 120-139 142 142 142 242 242 242 F F F 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 143 143 143 246 246 246 F F F 11 220-239 142 142 142 241 241 241 F F F 12 240-259 138 138 138 231 231 231 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 142 142 142 242 242 242 F F F 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 138 138 231 231 231 F F F 18 360-379 130 130 130 222 222 222 F F F 19 380-399 142 142 142 245 245 245 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 140 140 140 238 238 238 F F F 22 440-459 140 140 140 237 237 237 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 140 140 140 238 238 238 F F F 25 500-519 142 142 142 240 240 240 F F F 26 520-539 142 142 142 240 240 240 F F F 27 540-559 141 141 141 239 239 239 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 141 141 141 239 239 239 F F F Max gen: 143, Max flush: 246 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.201212967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:36.245313977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.245868816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.277600797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.278657747Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.278717764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.278736724Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.278763652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} {"msg":"Crucible 8509574d-80cb-45eb-9685-6cb7ae2a9fea has session id: 85b3459b-2dc4-4824-92e9-31179cc15163","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279382779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Upstairs opts: Upstairs UUID: 8509574d-80cb-45eb-9685-6cb7ae2a9fea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279425917Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Crucible stats registered with UUID: 8509574d-80cb-45eb-9685-6cb7ae2a9fea","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279450317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:36.279467426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279679978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279723596Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279748365Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279942788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.279964557Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} {"msg":"8509574d-80cb-45eb-9685-6cb7ae2a9fea active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280043194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280065243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"0","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280082562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"1","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280099542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"2","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280116181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.28013382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280172729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"ds_connection connected from Ok(127.0.0.1:37114)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280269965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"ds_connection connected from Ok(127.0.0.1:43506)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280311574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"ds_connection connected from Ok(127.0.0.1:57411)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.280345792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.28171663Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"1","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.281750089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"0","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.281770288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"2","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282365615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282396274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 238, 242, 234, 230, 237, 246, 241]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282440592Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 140, 136, 140, 142, 139, 137, 140, 143, 142]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282462522Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282482251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 238, 242, 234, 230, 237, 246, 241]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.28251541Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 140, 136, 140, 142, 139, 137, 140, 143, 142]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282534679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282554108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 238, 242, 234, 230, 237, 246, 241]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282572778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 140, 136, 140, 142, 139, 137, 140, 143, 142]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282591797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282614896Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282631465Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282648355Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282664344Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"All extents match","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282687613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"":"downstairs","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282704842Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282720022Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"0","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282736751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"1","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282753421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"client":"2","":"downstairs","session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.28277033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282786649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"8509574d-80cb-45eb-9685-6cb7ae2a9fea is now active with session: 85b3459b-2dc4-4824-92e9-31179cc15163","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282804409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282819568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788,"session_id":"85b3459b-2dc4-4824-92e9-31179cc15163"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282835548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} {"msg":"Activated sub_volume 8509574d-80cb-45eb-9685-6cb7ae2a9fea","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:36.282849277Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 32768 data: 14 11 10 12 10 8 12 14 02/30 Flush 03/30 Write block 481 len 28672 data: 9 10 14 15 16 15 18 04/30 Read block 208 len 20480 05/30 Read block 205 len 36864 06/30 Read block 542 len 32768 07/30 Write block 234 len 24576 data: 10 12 12 12 9 10 08/30 Read block 519 len 36864 09/30 Write block 107 len 12288 data: 9 7 7 10/30 Read block 40 len 40960 11/30 Read block 244 len 8192 12/30 Read block 93 len 8192 13/30 Write block 123 len 20480 data: 17 15 13 10 10 14/30 Write block 377 len 32768 data: 9 10 6 6 8 7 6 6 15/30 Read block 340 len 4096 16/30 Write block 545 len 40960 data: 9 10 10 12 11 12 12 11 12 13 17/30 Read block 63 len 16384 18/30 Read block 364 len 32768 19/30 Read block 274 len 36864 20/30 Read block 439 len 20480 21/30 Read block 16 len 36864 22/30 Read block 427 len 32768 23/30 Write block 389 len 16384 data: 10 12 10 11 24/30 Write block 249 len 12288 data: 8 11 11 25/30 Write block 345 len 40960 data: 11 10 11 13 10 10 9 10 9 8 26/30 Read block 106 len 4096 27/30 Read block 525 len 16384 28/30 Write block 451 len 20480 data: 8 6 8 7 6 29/30 Write block 584 len 36864 data: 11 12 12 11 8 7 8 8 8 30/30 Write block 220 len 20480 data: 7 7 11 12 11 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:36.303401374Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1788} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 140 140 140 238 238 238 F T T <--- 6 120-139 142 142 142 242 242 242 F T T <--- 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 143 143 143 246 246 246 F F F 11 220-239 142 142 142 241 241 241 F T T <--- 12 240-259 138 138 138 231 231 231 F T T <--- 13 260-279 138 138 138 231 231 231 F F F 14 280-299 142 142 142 242 242 242 F F F 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 138 138 138 231 231 231 F T T <--- 18 360-379 130 130 130 222 222 222 F T T <--- 19 380-399 142 142 142 245 245 245 F T T <--- 20 400-419 136 136 136 229 229 229 F F F 21 420-439 140 144 144 238 247 247 F F F <--- 22 440-459 140 140 140 237 237 237 F T T <--- 23 460-479 140 140 140 237 237 237 F F F 24 480-499 140 140 140 238 238 238 F T T <--- 25 500-519 142 142 142 240 240 240 F F F 26 520-539 142 142 142 240 240 240 F F F 27 540-559 141 141 141 239 239 239 F T T <--- 28 560-579 136 136 136 229 229 229 F F F 29 580-599 141 141 141 239 239 239 F T T <--- Max gen: 144, Max flush: 247 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 219 219 219 F F F 1 020-039 131 131 131 223 223 223 F F F 2 040-059 132 132 132 225 225 225 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 136 136 136 229 229 229 F F F 5 100-119 145 145 145 248 248 248 F F F 6 120-139 145 145 145 248 248 248 F F F 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 143 143 143 246 246 246 F F F 11 220-239 145 145 145 248 248 248 F F F 12 240-259 145 145 145 248 248 248 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 142 142 142 242 242 242 F F F 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 145 145 145 248 248 248 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 145 145 145 248 248 248 F F F 23 460-479 140 140 140 237 237 237 F F F 24 480-499 145 145 145 248 248 248 F F F 25 500-519 142 142 142 240 240 240 F F F 26 520-539 142 142 142 240 240 240 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 145, Max flush: 248 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.785571895Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.832340363Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.833166262Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.83321394Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.833235729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.833258158Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} {"msg":"Crucible 01d1b518-f28a-4a9e-87dc-26508213bdf1 has session id: f80b42e5-6ca5-48a7-b237-a0a0d3c3289d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.833927083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Upstairs opts: Upstairs UUID: 01d1b518-f28a-4a9e-87dc-26508213bdf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.83399647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Crucible stats registered with UUID: 01d1b518-f28a-4a9e-87dc-26508213bdf1","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834032059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:40.834059328Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834234301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834284609Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834304978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.8345173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834545179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} {"msg":"01d1b518-f28a-4a9e-87dc-26508213bdf1 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834612497Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834649825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"0","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834671215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"1","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834688774Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"2","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834705483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834723153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834752181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34442)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834905735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50037)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.834958793Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59007)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.83505041Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.836343521Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"0","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.836382959Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"1","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.836420218Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"2","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.836978687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837093412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[0]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 248, 248, 234, 230, 237, 246, 248]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837180069Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[0]R generation[0..12]: [128, 131, 132, 140, 136, 145, 145, 139, 137, 140, 143, 145]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837210738Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837232857Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[1]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 248, 248, 234, 230, 237, 246, 248]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837252166Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[1]R generation[0..12]: [128, 131, 132, 140, 136, 145, 145, 139, 137, 140, 143, 145]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837270845Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837290665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[2]R flush_numbers[0..12]: [219, 223, 225, 235, 229, 248, 248, 234, 230, 237, 246, 248]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837309564Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[2]R generation[0..12]: [128, 131, 132, 140, 136, 145, 145, 139, 137, 140, 143, 145]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837328293Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837354592Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837384811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.83741941Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837443099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837459868Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"":"downstairs","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837476648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837491727Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"0","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837511406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"1","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837528196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"client":"2","":"downstairs","session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837545175Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837561064Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"01d1b518-f28a-4a9e-87dc-26508213bdf1 is now active with session: f80b42e5-6ca5-48a7-b237-a0a0d3c3289d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837578914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837594073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796,"session_id":"f80b42e5-6ca5-48a7-b237-a0a0d3c3289d"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837609903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} {"msg":"Activated sub_volume 01d1b518-f28a-4a9e-87dc-26508213bdf1","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:40.837623712Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 28672 data: 13 14 14 12 13 13 15 02/30 Flush 03/30 Write block 251 len 12288 data: 12 12 13 04/30 Read block 488 len 8192 05/30 Read block 483 len 20480 06/30 Write block 205 len 4096 data: 15 07/30 Read block 231 len 20480 08/30 Write block 461 len 40960 data: 8 10 10 8 9 7 7 6 5 6 09/30 Read block 416 len 36864 10/30 Read block 558 len 20480 11/30 Write block 7 len 24576 data: 9 10 10 10 10 12 12/30 Write block 80 len 28672 data: 8 7 9 11 12 13 15 13/30 Read block 140 len 36864 14/30 Flush 15/30 Flush 16/30 Write block 30 len 32768 data: 10 9 10 8 7 6 5 6 17/30 Write block 8 len 32768 data: 11 11 11 11 13 10 10 10 18/30 Read block 436 len 32768 19/30 Read block 256 len 32768 20/30 Read block 233 len 36864 21/30 Read block 547 len 24576 22/30 Read block 564 len 36864 23/30 Read block 259 len 40960 24/30 Read block 134 len 12288 25/30 Read block 128 len 16384 26/30 Read block 356 len 4096 27/30 Write block 530 len 4096 data: 10 28/30 Read block 236 len 8192 29/30 Write block 383 len 36864 data: 7 7 8 7 7 9 11 13 11 30/30 Write block 395 len 20480 data: 16 16 15 14 12 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Read 2 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 27 31 Done 26 26 1 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:40.85764152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1796} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 128 250 250 219 T T F <--- 1 020-039 131 131 131 223 223 223 T T F <--- 2 040-059 146 146 146 249 249 249 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 146 146 136 250 250 229 F F F <--- 5 100-119 145 145 145 248 248 248 F F F 6 120-139 145 145 145 248 248 248 F F F 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 146 146 143 250 250 246 F F F <--- 11 220-239 145 145 145 248 248 248 F F F 12 240-259 146 146 145 250 250 248 F F T <--- 13 260-279 138 138 138 231 231 231 F F F 14 280-299 142 142 142 242 242 242 F F F 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 145 145 145 248 248 248 T T F <--- 20 400-419 136 136 136 229 229 229 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 145 145 145 248 248 248 F F F 23 460-479 146 146 140 250 250 237 F F F <--- 24 480-499 145 145 145 248 248 248 F F F 25 500-519 142 142 142 240 240 240 F F F 26 520-539 142 142 142 240 240 240 T T F <--- 27 540-559 145 145 145 248 248 248 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 146, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 251 251 251 F F F 1 020-039 147 147 147 251 251 251 F F F 2 040-059 146 146 146 249 249 249 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 146 146 146 250 250 250 F F F 5 100-119 145 145 145 248 248 248 F F F 6 120-139 145 145 145 248 248 248 F F F 7 140-159 139 139 139 234 234 234 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 146 146 146 250 250 250 F F F 11 220-239 145 145 145 248 248 248 F F F 12 240-259 146 146 146 250 250 250 F F F 13 260-279 138 138 138 231 231 231 F F F 14 280-299 142 142 142 242 242 242 F F F 15 300-319 140 140 140 237 237 237 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 147 147 147 251 251 251 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 145 145 145 248 248 248 F F F 23 460-479 146 146 146 250 250 250 F F F 24 480-499 145 145 145 248 248 248 F F F 25 500-519 142 142 142 240 240 240 F F F 26 520-539 147 147 147 251 251 251 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 147, Max flush: 251 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.259829687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:45.304773165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.30517155Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.336644781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.337351934Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.337400622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.337417001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.337439111Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} {"msg":"Crucible aba05ded-a73f-4d50-989f-2cdf01ca379d has session id: f46b2338-2e1c-430c-9c96-9f2f6fe7f706","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338015258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Upstairs opts: Upstairs UUID: aba05ded-a73f-4d50-989f-2cdf01ca379d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338068447Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Crucible stats registered with UUID: aba05ded-a73f-4d50-989f-2cdf01ca379d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338100885Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:45.338127204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338348126Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338391104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338411023Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338554378Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338574337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} {"msg":"aba05ded-a73f-4d50-989f-2cdf01ca379d active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338649064Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338669684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"0","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338693493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"1","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338725931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"2","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.33875933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338782679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338803339Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"ds_connection connected from Ok(127.0.0.1:34952)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338887525Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"ds_connection connected from Ok(127.0.0.1:61588)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338942813Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"ds_connection connected from Ok(127.0.0.1:54702)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.338978162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.340328221Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"1","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.340374019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"2","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.340412667Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"0","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.340910778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.340996595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[0]R flush_numbers[0..12]: [251, 251, 249, 235, 250, 248, 248, 234, 230, 237, 250, 248]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341110131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[0]R generation[0..12]: [147, 147, 146, 140, 146, 145, 145, 139, 137, 140, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.34113829Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341158889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[1]R flush_numbers[0..12]: [251, 251, 249, 235, 250, 248, 248, 234, 230, 237, 250, 248]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341178078Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[1]R generation[0..12]: [147, 147, 146, 140, 146, 145, 145, 139, 137, 140, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341196717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341216147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[2]R flush_numbers[0..12]: [251, 251, 249, 235, 250, 248, 248, 234, 230, 237, 250, 248]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341238756Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[2]R generation[0..12]: [147, 147, 146, 140, 146, 145, 145, 139, 137, 140, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341257935Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341277494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341293914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Generation requested: 148 >= found:148","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341319283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341335742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341351702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"":"downstairs","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341369001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.34138411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"0","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341400749Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"1","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341417349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"client":"2","":"downstairs","session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341434568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341450598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"aba05ded-a73f-4d50-989f-2cdf01ca379d is now active with session: f46b2338-2e1c-430c-9c96-9f2f6fe7f706","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341468527Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341483797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804,"session_id":"f46b2338-2e1c-430c-9c96-9f2f6fe7f706"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341501786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} {"msg":"Activated sub_volume aba05ded-a73f-4d50-989f-2cdf01ca379d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:45.341515655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 446 len 28672 data: 12 11 10 8 7 9 7 02/30 Flush 03/30 Write block 519 len 16384 data: 15 15 12 14 04/30 Write block 200 len 20480 data: 15 15 13 11 14 05/30 Write block 273 len 32768 data: 12 15 12 11 10 9 7 7 06/30 Write block 287 len 32768 data: 9 9 9 9 10 11 12 12 07/30 Write block 144 len 8192 data: 11 9 08/30 Write block 152 len 4096 data: 13 09/30 Write block 526 len 36864 data: 10 8 7 9 11 10 10 10 12 10/30 Read block 213 len 12288 11/30 Read block 88 len 36864 12/30 Write block 303 len 16384 data: 17 14 16 15 13/30 Read block 564 len 16384 14/30 Read block 153 len 4096 15/30 Write block 230 len 36864 data: 11 12 12 10 11 13 13 13 10 16/30 Read block 235 len 40960 17/30 Write block 472 len 20480 data: 9 7 7 5 6 18/30 Flush 19/30 Write block 155 len 32768 data: 13 12 11 11 14 12 13 12 20/30 Read block 90 len 4096 21/30 Read block 312 len 24576 22/30 Write block 432 len 16384 data: 11 9 13 15 23/30 Write block 292 len 32768 data: 12 13 13 14 13 13 16 16 24/30 Read block 197 len 20480 25/30 Read block 594 len 4096 26/30 Write block 499 len 32768 data: 7 8 8 13 16 14 14 12 27/30 Read block 234 len 8192 28/30 Read block 169 len 16384 29/30 Write block 430 len 8192 data: 11 13 30/30 Read block 522 len 40960 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:45.361397598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1804} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 251 251 251 F F F 1 020-039 147 147 147 251 251 251 F F F 2 040-059 146 146 146 249 249 249 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 146 146 146 250 250 250 F F F 5 100-119 145 145 145 248 248 248 F F F 6 120-139 145 145 145 248 248 248 F F F 7 140-159 139 148 148 234 253 253 F T T <--- 8 160-179 137 137 137 230 230 230 F T T <--- 9 180-199 140 140 140 237 237 237 F F F 10 200-219 146 148 148 250 253 253 F F F <--- 11 220-239 145 148 148 248 253 253 F F F <--- 12 240-259 146 146 146 250 250 250 F F F 13 260-279 138 148 148 231 253 253 F F F <--- 14 280-299 142 148 148 242 253 253 F T T <--- 15 300-319 140 148 148 237 253 253 F F F <--- 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 147 147 147 251 251 251 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 144 144 144 247 247 247 F T T <--- 22 440-459 145 148 148 248 252 252 F F F <--- 23 460-479 146 148 148 250 253 253 F F F <--- 24 480-499 145 145 145 248 248 248 F T T <--- 25 500-519 142 148 148 240 253 253 F T T <--- 26 520-539 147 148 148 251 253 253 F F F <--- 27 540-559 145 145 145 248 248 248 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 148, Max flush: 253 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 251 251 251 F F F 1 020-039 147 147 147 251 251 251 F F F 2 040-059 146 146 146 249 249 249 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 146 146 146 250 250 250 F F F 5 100-119 145 145 145 248 248 248 F F F 6 120-139 145 145 145 248 248 248 F F F 7 140-159 149 149 149 254 254 254 F F F 8 160-179 149 149 149 254 254 254 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 148 148 148 253 253 253 F F F 11 220-239 148 148 148 253 253 253 F F F 12 240-259 146 146 146 250 250 250 F F F 13 260-279 148 148 148 253 253 253 F F F 14 280-299 149 149 149 254 254 254 F F F 15 300-319 148 148 148 253 253 253 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 147 147 147 251 251 251 F F F 20 400-419 136 136 136 229 229 229 F F F 21 420-439 149 149 149 254 254 254 F F F 22 440-459 148 148 148 252 252 252 F F F 23 460-479 148 148 148 253 253 253 F F F 24 480-499 149 149 149 254 254 254 F F F 25 500-519 149 149 149 254 254 254 F F F 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 136 136 136 229 229 229 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 149, Max flush: 254 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.809770095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:49.852775947Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.853135444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.884313076Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885042668Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885101316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885121065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885148634Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} {"msg":"Crucible 6b45d88b-efb2-48b8-97f4-8c528b5091a9 has session id: f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885695873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Upstairs opts: Upstairs UUID: 6b45d88b-efb2-48b8-97f4-8c528b5091a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885745192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Crucible stats registered with UUID: 6b45d88b-efb2-48b8-97f4-8c528b5091a9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885766571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:49.88578309Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.885978083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886031761Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.88605122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886181185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886200734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} {"msg":"6b45d88b-efb2-48b8-97f4-8c528b5091a9 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886266882Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886286451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"0","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.88630891Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"1","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886338159Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"2","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886369148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886402556Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886437355Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49423)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886492443Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61182)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886527992Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60864)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.886617788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.887851131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"2","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.88789086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"1","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888045274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"0","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888552425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888582653Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[0]R flush_numbers[0..12]: [251, 251, 249, 235, 250, 248, 248, 254, 254, 237, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888601363Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[0]R generation[0..12]: [147, 147, 146, 140, 146, 145, 145, 149, 149, 140, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888621012Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888641191Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[1]R flush_numbers[0..12]: [251, 251, 249, 235, 250, 248, 248, 254, 254, 237, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888660101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[1]R generation[0..12]: [147, 147, 146, 140, 146, 145, 145, 149, 149, 140, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.88867904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888698499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[2]R flush_numbers[0..12]: [251, 251, 249, 235, 250, 248, 248, 254, 254, 237, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888717288Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[2]R generation[0..12]: [147, 147, 146, 140, 146, 145, 145, 149, 149, 140, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888736567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888756007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888772196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888792725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888816635Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888840943Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"":"downstairs","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888858003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888873142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"0","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888889702Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"1","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888906191Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"client":"2","":"downstairs","session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.88892344Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.88893964Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"6b45d88b-efb2-48b8-97f4-8c528b5091a9 is now active with session: f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888957799Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888973018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812,"session_id":"f38b2fcb-19e9-48b4-93cd-9dd4da8c0ef6"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.888989018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} {"msg":"Activated sub_volume 6b45d88b-efb2-48b8-97f4-8c528b5091a9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:49.889002777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 427 len 12288 data: 16 15 12 02/30 Write block 496 len 36864 data: 12 9 9 8 9 9 14 17 15 03/30 Read block 151 len 4096 04/30 Read block 349 len 40960 05/30 Write block 102 len 36864 data: 14 14 12 10 9 10 8 8 6 06/30 Read block 66 len 12288 07/30 Read block 231 len 12288 08/30 Read block 446 len 20480 09/30 Read block 185 len 12288 10/30 Flush 11/30 Write block 390 len 16384 data: 14 12 12 15 12/30 Read block 202 len 24576 13/30 Write block 35 len 4096 data: 7 14/30 Read block 473 len 20480 15/30 Read block 168 len 16384 16/30 Read block 57 len 36864 17/30 Write block 126 len 8192 data: 11 11 18/30 Write block 25 len 16384 data: 14 13 12 15 19/30 Write block 100 len 28672 data: 14 15 15 15 13 11 10 20/30 Write block 436 len 32768 data: 14 14 15 16 15 13 10 10 21/30 Write block 253 len 40960 data: 14 10 10 10 9 10 11 9 12 11 22/30 Write block 572 len 28672 data: 15 17 17 19 18 15 13 23/30 Read block 427 len 28672 24/30 Read block 110 len 20480 25/30 Read block 307 len 8192 26/30 Write block 395 len 24576 data: 17 17 16 15 13 12 27/30 Write block 431 len 4096 data: 14 28/30 Write block 37 len 36864 data: 7 10 10 12 13 12 13 11 9 29/30 Read block 7 len 24576 30/30 Write block 201 len 32768 data: 16 14 12 15 16 13 15 15 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:49.907782312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1812} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 251 251 251 F F F 1 020-039 147 147 147 251 251 251 F T T <--- 2 040-059 146 146 146 249 249 249 F T T <--- 3 060-079 140 140 140 235 235 235 F F F 4 080-099 146 146 146 250 250 250 F F F 5 100-119 145 150 150 248 255 255 F T T <--- 6 120-139 145 145 145 248 248 248 F T T <--- 7 140-159 149 149 149 254 254 254 F F F 8 160-179 149 149 149 254 254 254 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 148 148 148 253 253 253 F T T <--- 11 220-239 148 148 148 253 253 253 F F F 12 240-259 146 146 146 250 250 250 F T T <--- 13 260-279 148 148 148 253 253 253 F T T <--- 14 280-299 149 149 149 254 254 254 F F F 15 300-319 148 148 148 253 253 253 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 147 147 147 251 251 251 F T T <--- 20 400-419 136 136 136 229 229 229 F T T <--- 21 420-439 149 150 150 254 255 255 T T T <--- 22 440-459 148 148 148 252 252 252 F T T <--- 23 460-479 148 148 148 253 253 253 F F F 24 480-499 149 150 150 254 255 255 F F F <--- 25 500-519 149 150 150 254 255 255 F F F <--- 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 136 136 136 229 229 229 F T T <--- 29 580-599 145 145 145 248 248 248 F F F Max gen: 150, Max flush: 255 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 251 251 251 F F F 1 020-039 151 151 151 256 256 256 F F F 2 040-059 151 151 151 256 256 256 F F F 3 060-079 140 140 140 235 235 235 F F F 4 080-099 146 146 146 250 250 250 F F F 5 100-119 151 151 151 256 256 256 F F F 6 120-139 151 151 151 256 256 256 F F F 7 140-159 149 149 149 254 254 254 F F F 8 160-179 149 149 149 254 254 254 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 151 151 151 256 256 256 F F F 11 220-239 148 148 148 253 253 253 F F F 12 240-259 151 151 151 256 256 256 F F F 13 260-279 151 151 151 256 256 256 F F F 14 280-299 149 149 149 254 254 254 F F F 15 300-319 148 148 148 253 253 253 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 151 151 151 256 256 256 F F F 20 400-419 151 151 151 256 256 256 F F F 21 420-439 151 151 151 256 256 256 F F F 22 440-459 151 151 151 256 256 256 F F F 23 460-479 148 148 148 253 253 253 F F F 24 480-499 150 150 150 255 255 255 F F F 25 500-519 150 150 150 255 255 255 F F F 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 151 151 151 256 256 256 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 151, Max flush: 256 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.371729137Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:54.416172504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.416578359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.447241831Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.447905466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.447976403Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448011871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.44804116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} {"msg":"Crucible c81d017f-2c6a-45bc-b582-d8fa8131396b has session id: 6a241848-b3dd-405c-8450-eee18c211f0d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448554631Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Upstairs opts: Upstairs UUID: c81d017f-2c6a-45bc-b582-d8fa8131396b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448620628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Crucible stats registered with UUID: c81d017f-2c6a-45bc-b582-d8fa8131396b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448656897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:54.448687446Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448864099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448913487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.448935876Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449090241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449112009Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} {"msg":"c81d017f-2c6a-45bc-b582-d8fa8131396b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449177907Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449202056Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"0","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449228975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"1","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449261284Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"2","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449293283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449325661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.44936083Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48583)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449396519Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61289)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449431428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35513)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.449505095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.450623742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"2","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.450657801Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"1","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.45067786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"0","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451298106Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451327185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[0]R flush_numbers[0..12]: [251, 256, 256, 235, 250, 256, 256, 254, 254, 237, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451354114Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[0]R generation[0..12]: [147, 151, 151, 140, 146, 151, 151, 149, 149, 140, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451378723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451398822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[1]R flush_numbers[0..12]: [251, 256, 256, 235, 250, 256, 256, 254, 254, 237, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451417972Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[1]R generation[0..12]: [147, 151, 151, 140, 146, 151, 151, 149, 149, 140, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451436711Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.45145618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[2]R flush_numbers[0..12]: [251, 256, 256, 235, 250, 256, 256, 254, 254, 237, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451474919Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[2]R generation[0..12]: [147, 151, 151, 140, 146, 151, 151, 149, 149, 140, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451493489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451512938Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451529377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451546367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451562336Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451586715Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"":"downstairs","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451603735Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451618714Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"0","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451638213Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"1","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451654943Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"client":"2","":"downstairs","session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451671772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451687962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"c81d017f-2c6a-45bc-b582-d8fa8131396b is now active with session: 6a241848-b3dd-405c-8450-eee18c211f0d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451705861Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.45172089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820,"session_id":"6a241848-b3dd-405c-8450-eee18c211f0d"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.45173684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} {"msg":"Activated sub_volume c81d017f-2c6a-45bc-b582-d8fa8131396b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:54.451750779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 77 len 32768 data: 10 11 9 9 8 10 12 13 02/30 Flush 03/30 Write block 286 len 24576 data: 9 10 10 10 10 11 04/30 Read block 466 len 24576 05/30 Write block 411 len 32768 data: 13 12 11 12 8 8 10 10 06/30 Read block 78 len 28672 07/30 Write block 409 len 20480 data: 13 12 14 13 12 08/30 Write block 151 len 12288 data: 13 14 15 09/30 Read block 241 len 32768 10/30 Read block 365 len 16384 11/30 Read block 412 len 28672 12/30 Read block 257 len 20480 13/30 Flush 14/30 Write block 460 len 36864 data: 7 9 11 11 9 10 8 8 7 15/30 Flush 16/30 Write block 14 len 32768 data: 11 11 12 14 11 12 15 18 17/30 Read block 154 len 24576 18/30 Read block 489 len 8192 19/30 Read block 347 len 32768 20/30 Write block 571 len 28672 data: 17 16 18 18 20 19 16 21/30 Read block 344 len 24576 22/30 Read block 270 len 36864 23/30 Read block 338 len 4096 24/30 Read block 480 len 28672 25/30 Read block 414 len 32768 26/30 Write block 500 len 8192 data: 10 10 27/30 Write block 312 len 12288 data: 15 14 12 28/30 Read block 591 len 28672 29/30 Read block 311 len 28672 30/30 Write block 85 len 36864 data: 14 16 13 13 13 9 8 10 9 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 5 27 27 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:54.474074109Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1820} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 251 251 251 F T T <--- 1 020-039 151 151 151 256 256 256 F T T <--- 2 040-059 151 151 151 256 256 256 F F F 3 060-079 152 152 152 257 257 257 F F F 4 080-099 152 152 152 257 257 257 F T T <--- 5 100-119 151 151 151 256 256 256 F F F 6 120-139 151 151 151 256 256 256 F F F 7 140-159 149 152 152 254 258 258 F F F <--- 8 160-179 149 149 149 254 254 254 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 151 151 151 256 256 256 F F F 11 220-239 148 148 148 253 253 253 F F F 12 240-259 151 151 151 256 256 256 F F F 13 260-279 151 151 151 256 256 256 F F F 14 280-299 149 152 152 254 258 258 T F F <--- 15 300-319 148 148 148 253 253 253 F T T <--- 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 151 151 151 256 256 256 F F F 20 400-419 151 152 152 256 258 258 T F F <--- 21 420-439 151 151 151 256 256 256 F F F 22 440-459 151 151 151 256 256 256 F F F 23 460-479 148 152 152 253 259 259 F F F <--- 24 480-499 150 150 150 255 255 255 F F F 25 500-519 150 150 150 255 255 255 F T T <--- 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 151 151 151 256 256 256 F T T <--- 29 580-599 145 145 145 248 248 248 F F F Max gen: 152, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 153 153 153 260 260 260 F F F 2 040-059 151 151 151 256 256 256 F F F 3 060-079 152 152 152 257 257 257 F F F 4 080-099 153 153 153 260 260 260 F F F 5 100-119 151 151 151 256 256 256 F F F 6 120-139 151 151 151 256 256 256 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 149 149 149 254 254 254 F F F 9 180-199 140 140 140 237 237 237 F F F 10 200-219 151 151 151 256 256 256 F F F 11 220-239 148 148 148 253 253 253 F F F 12 240-259 151 151 151 256 256 256 F F F 13 260-279 151 151 151 256 256 256 F F F 14 280-299 152 152 152 258 258 258 F F F 15 300-319 153 153 153 260 260 260 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 145 145 248 248 248 F F F 19 380-399 151 151 151 256 256 256 F F F 20 400-419 152 152 152 258 258 258 F F F 21 420-439 151 151 151 256 256 256 F F F 22 440-459 151 151 151 256 256 256 F F F 23 460-479 152 152 152 259 259 259 F F F 24 480-499 150 150 150 255 255 255 F F F 25 500-519 153 153 153 260 260 260 F F F 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 153 153 153 260 260 260 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 153, Max flush: 260 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.905626887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:58.949750706Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.950144461Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.982306656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.983177253Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.983224501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.983240161Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.98326203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} {"msg":"Crucible 9039995a-64d9-47ab-b72a-13b1f8afea74 has session id: 036369cf-f992-4439-8ec4-16b5df1102f2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.983819859Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Upstairs opts: Upstairs UUID: 9039995a-64d9-47ab-b72a-13b1f8afea74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.983881176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Crucible stats registered with UUID: 9039995a-64d9-47ab-b72a-13b1f8afea74","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.983906815Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:58.983923215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984127707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984177075Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984209214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984371247Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984395747Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} {"msg":"9039995a-64d9-47ab-b72a-13b1f8afea74 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984459214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984489603Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"0","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984511562Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"1","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984533151Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"2","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984553651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.98457929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984616858Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34023)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984662416Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37512)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984707905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36975)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.984751003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986116341Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"1","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986162739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"2","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986200538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"0","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986781156Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986815305Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[0]R flush_numbers[0..12]: [260, 260, 256, 257, 260, 256, 256, 258, 254, 237, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986846423Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[0]R generation[0..12]: [153, 153, 151, 152, 153, 151, 151, 152, 149, 140, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986871372Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986895921Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[1]R flush_numbers[0..12]: [260, 260, 256, 257, 260, 256, 256, 258, 254, 237, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.98691945Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[1]R generation[0..12]: [153, 153, 151, 152, 153, 151, 151, 152, 149, 140, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986943089Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986967719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[2]R flush_numbers[0..12]: [260, 260, 256, 257, 260, 256, 256, 258, 254, 237, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.986996378Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[2]R generation[0..12]: [153, 153, 151, 152, 153, 151, 151, 152, 149, 140, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987020487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987044556Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987074415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987096554Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987116543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987136432Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","Wait for a query_work_queue command to finish before sending IO pid":1828,"":"downstairs","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987164771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987183871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"0","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.98720472Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"1","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987225349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"client":"2","":"downstairs","session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987246698Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987266567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"9039995a-64d9-47ab-b72a-13b1f8afea74 is now active with session: 036369cf-f992-4439-8ec4-16b5df1102f2","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987288967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987307846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828,"session_id":"036369cf-f992-4439-8ec4-16b5df1102f2"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987332235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} {"msg":"Activated sub_volume 9039995a-64d9-47ab-b72a-13b1f8afea74","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:03:58.987349944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 24576 data: 11 10 10 14 10 11 02/30 Write block 311 len 36864 data: 14 16 15 13 11 9 11 12 15 03/30 Write block 309 len 40960 data: 16 12 15 17 16 14 12 10 12 13 04/30 Write block 504 len 8192 data: 16 15 05/30 Read block 50 len 12288 06/30 Flush 07/30 Flush 08/30 Write block 87 len 8192 data: 14 14 09/30 Write block 397 len 12288 data: 17 16 14 10/30 Read block 154 len 4096 11/30 Read block 512 len 12288 12/30 Write block 126 len 4096 data: 12 13/30 Write block 276 len 4096 data: 12 14/30 Read block 378 len 40960 15/30 Flush 16/30 Write block 475 len 36864 data: 6 7 7 6 8 7 10 11 15 17/30 Read block 468 len 24576 18/30 Read block 412 len 28672 19/30 Write block 379 len 36864 data: 7 7 9 8 8 8 9 8 8 20/30 Write block 573 len 12288 data: 19 19 21 21/30 Flush 22/30 Write block 74 len 4096 data: 12 23/30 Flush 24/30 Write block 176 len 40960 data: 12 10 11 10 6 4 5 4 6 10 25/30 Read block 514 len 28672 26/30 Read block 214 len 40960 27/30 Write block 372 len 36864 data: 13 12 10 11 10 10 11 8 8 28/30 Write block 563 len 8192 data: 15 12 29/30 Read block 269 len 40960 30/30 Write block 123 len 8192 data: 18 16 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:03:59.006144368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1828} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 153 153 153 260 260 260 F F F 2 040-059 151 151 151 256 256 256 F F F 3 060-079 152 154 154 257 265 265 F F F <--- 4 080-099 153 154 154 260 263 263 F F F <--- 5 100-119 151 151 151 256 256 256 F F F 6 120-139 151 154 154 256 263 263 F T T <--- 7 140-159 152 152 152 258 258 258 F F F 8 160-179 149 149 149 254 254 254 F T T <--- 9 180-199 140 140 140 237 237 237 F T T <--- 10 200-219 151 151 151 256 256 256 F F F 11 220-239 148 148 148 253 253 253 F F F 12 240-259 151 151 151 256 256 256 F F F 13 260-279 151 154 154 256 263 263 F F F <--- 14 280-299 152 152 152 258 258 258 F F F 15 300-319 153 154 154 260 261 261 T F F <--- 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 145 154 154 248 264 264 T T T <--- 19 380-399 151 154 154 256 264 264 F T T <--- 20 400-419 152 152 152 258 258 258 F F F 21 420-439 151 151 151 256 256 256 F F F 22 440-459 151 151 151 256 256 256 F F F 23 460-479 152 154 154 259 264 264 F F F <--- 24 480-499 150 154 154 255 264 264 F F F <--- 25 500-519 153 154 154 260 261 261 T F F <--- 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 153 154 154 260 264 264 F T T <--- 29 580-599 145 145 145 248 248 248 F F F Max gen: 154, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 153 153 153 260 260 260 F F F 2 040-059 151 151 151 256 256 256 F F F 3 060-079 154 154 154 265 265 265 F F F 4 080-099 154 154 154 263 263 263 F F F 5 100-119 151 151 151 256 256 256 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 155 155 155 266 266 266 F F F 10 200-219 151 151 151 256 256 256 F F F 11 220-239 148 148 148 253 253 253 F F F 12 240-259 151 151 151 256 256 256 F F F 13 260-279 154 154 154 263 263 263 F F F 14 280-299 152 152 152 258 258 258 F F F 15 300-319 154 154 154 261 261 261 F F F 16 320-339 135 135 135 228 228 228 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 155 155 155 266 266 266 F F F 19 380-399 155 155 155 266 266 266 F F F 20 400-419 152 152 152 258 258 258 F F F 21 420-439 151 151 151 256 256 256 F F F 22 440-459 151 151 151 256 256 256 F F F 23 460-479 154 154 154 264 264 264 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 148 148 148 253 253 253 F F F 27 540-559 145 145 145 248 248 248 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 155, Max flush: 266 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.454317444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:03.497845366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.498412185Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.529994462Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.530883878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.530941386Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.530961045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.530989014Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} {"msg":"Crucible 2cf23ada-b612-41ab-9860-c07c440b018f has session id: c8414f9a-77a2-4226-a842-40a961c53b36","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.531560302Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Upstairs opts: Upstairs UUID: 2cf23ada-b612-41ab-9860-c07c440b018f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.531623029Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Crucible stats registered with UUID: 2cf23ada-b612-41ab-9860-c07c440b018f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.531658988Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:03.531678867Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.53187125Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.531928608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.531962007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532091212Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532115981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} {"msg":"2cf23ada-b612-41ab-9860-c07c440b018f active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532184068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532203798Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"0","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532221677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"1","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532239366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"2","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532263555Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532296474Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532329233Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"ds_connection connected from Ok(127.0.0.1:60652)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532361261Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"ds_connection connected from Ok(127.0.0.1:61593)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.53240053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"ds_connection connected from Ok(127.0.0.1:61029)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.532644301Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.53372199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"1","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.533848095Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"0","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.533911652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"2","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534372375Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534469441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[0]R flush_numbers[0..12]: [260, 260, 256, 265, 263, 256, 266, 258, 266, 266, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534541518Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[0]R generation[0..12]: [153, 153, 151, 154, 154, 151, 155, 152, 155, 155, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534565848Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534585987Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[1]R flush_numbers[0..12]: [260, 260, 256, 265, 263, 256, 266, 258, 266, 266, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534605346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[1]R generation[0..12]: [153, 153, 151, 154, 154, 151, 155, 152, 155, 155, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534624235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534643725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[2]R flush_numbers[0..12]: [260, 260, 256, 265, 263, 256, 266, 258, 266, 266, 256, 253]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534662474Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[2]R generation[0..12]: [153, 153, 151, 154, 154, 151, 155, 152, 155, 155, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534681403Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534700732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534716912Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534737811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.5347605Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534777189Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"":"downstairs","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534794279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534809508Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"0","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534826347Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"1","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534843167Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"client":"2","":"downstairs","session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534860146Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534876306Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"2cf23ada-b612-41ab-9860-c07c440b018f is now active with session: c8414f9a-77a2-4226-a842-40a961c53b36","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534894245Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534909434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836,"session_id":"c8414f9a-77a2-4226-a842-40a961c53b36"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534925484Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} {"msg":"Activated sub_volume 2cf23ada-b612-41ab-9860-c07c440b018f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:03.534939433Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 535 len 32768 data: 12 13 13 12 11 8 10 9 02/30 Write block 553 len 32768 data: 13 14 12 14 17 17 20 16 03/30 Write block 115 len 16384 data: 12 14 17 17 04/30 Read block 388 len 40960 05/30 Read block 372 len 32768 06/30 Read block 503 len 8192 07/30 Write block 462 len 28672 data: 12 12 10 11 9 9 8 08/30 Write block 41 len 40960 data: 14 13 14 12 10 10 14 15 15 13 09/30 Write block 217 len 36864 data: 10 8 6 8 8 12 13 12 8 10/30 Flush 11/30 Write block 241 len 4096 data: 7 12/30 Flush 13/30 Write block 201 len 24576 data: 17 15 13 16 17 14 14/30 Read block 243 len 24576 15/30 Write block 62 len 12288 data: 14 11 12 16/30 Flush 17/30 Write block 426 len 36864 data: 13 17 16 13 12 15 12 10 14 18/30 Read block 75 len 24576 19/30 Flush 20/30 Write block 332 len 12288 data: 8 9 8 21/30 Read block 449 len 40960 22/30 Read block 551 len 24576 23/30 Read block 349 len 4096 24/30 Write block 325 len 28672 data: 8 8 7 8 8 9 9 25/30 Write block 33 len 4096 data: 9 26/30 Write block 63 len 8192 data: 12 13 27/30 Read block 132 len 8192 28/30 Read block 283 len 24576 29/30 Read block 459 len 12288 30/30 Write block 287 len 12288 data: 11 11 11 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 7 29 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:03.554866494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1836} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 153 153 153 260 260 260 F T T <--- 2 040-059 151 156 156 256 267 267 F F F <--- 3 060-079 154 156 156 265 269 269 F T T <--- 4 080-099 154 154 154 263 263 263 F F F 5 100-119 151 156 156 256 267 267 T F F <--- 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 155 155 155 266 266 266 F F F 10 200-219 151 156 156 256 269 269 F F F <--- 11 220-239 148 156 156 253 267 267 F F F <--- 12 240-259 151 156 156 256 268 268 F F F <--- 13 260-279 154 154 154 263 263 263 F F F 14 280-299 152 152 152 258 258 258 F T T <--- 15 300-319 154 154 154 261 261 261 F F F 16 320-339 135 135 135 228 228 228 F T T <--- 17 340-359 145 145 145 248 248 248 F F F 18 360-379 155 155 155 266 266 266 F F F 19 380-399 155 155 155 266 266 266 F F F 20 400-419 152 152 152 258 258 258 F F F 21 420-439 151 156 156 256 270 270 F F F <--- 22 440-459 151 151 151 256 256 256 F F F 23 460-479 154 156 156 264 267 267 T F F <--- 24 480-499 154 154 154 264 264 264 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 148 156 156 253 267 267 T F F <--- 27 540-559 145 156 156 248 267 267 T F F <--- 28 560-579 155 156 156 266 267 267 T F F <--- 29 580-599 145 145 145 248 248 248 F F F Max gen: 156, Max flush: 270 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 156 156 156 267 267 267 F F F 3 060-079 157 157 157 271 271 271 F F F 4 080-099 154 154 154 263 263 263 F F F 5 100-119 156 156 156 267 267 267 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 155 155 155 266 266 266 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 156 156 156 268 268 268 F F F 13 260-279 154 154 154 263 263 263 F F F 14 280-299 157 157 157 271 271 271 F F F 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 155 155 155 266 266 266 F F F 19 380-399 155 155 155 266 266 266 F F F 20 400-419 152 152 152 258 258 258 F F F 21 420-439 156 156 156 270 270 270 F F F 22 440-459 151 151 151 256 256 256 F F F 23 460-479 156 156 156 267 267 267 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 156 156 156 267 267 267 F F F 27 540-559 156 156 156 267 267 267 F F F 28 560-579 156 156 156 267 267 267 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 157, Max flush: 271 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.003594218Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:08.046572411Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.046941967Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.077797812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.078479116Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.078538503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.078557783Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.078585082Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} {"msg":"Crucible 2d70bda5-751e-4f52-bf16-868e162aeec3 has session id: c0baf1da-646f-41aa-b1ba-47c71c95becb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079104892Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Upstairs opts: Upstairs UUID: 2d70bda5-751e-4f52-bf16-868e162aeec3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.07915346Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Crucible stats registered with UUID: 2d70bda5-751e-4f52-bf16-868e162aeec3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079183249Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:08.079203488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079384731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.07942204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079440789Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079625502Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079647971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} {"msg":"2d70bda5-751e-4f52-bf16-868e162aeec3 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079709129Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079732628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"0","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079750087Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"1","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079766567Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"2","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079789036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079820065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079850913Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36363)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079948Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"ds_connection connected from Ok(127.0.0.1:54043)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.079995568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55074)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.080117993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.081157944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"1","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.081303978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"2","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.081407244Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"0","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.081809489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.081892566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[0]R flush_numbers[0..12]: [260, 271, 267, 271, 263, 267, 266, 258, 266, 266, 269, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082012871Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[0]R generation[0..12]: [153, 157, 156, 157, 154, 156, 155, 152, 155, 155, 156, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.08204351Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082064119Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[1]R flush_numbers[0..12]: [260, 271, 267, 271, 263, 267, 266, 258, 266, 266, 269, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082083108Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[1]R generation[0..12]: [153, 157, 156, 157, 154, 156, 155, 152, 155, 155, 156, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082101728Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082121257Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[2]R flush_numbers[0..12]: [260, 271, 267, 271, 263, 267, 266, 258, 266, 266, 269, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082140226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[2]R generation[0..12]: [153, 157, 156, 157, 154, 156, 155, 152, 155, 155, 156, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082158916Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082178245Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082194674Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082218763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082234843Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082250732Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"":"downstairs","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082267692Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082283071Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"0","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.08230029Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"1","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.08231953Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"client":"2","":"downstairs","session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082336569Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082352658Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"2d70bda5-751e-4f52-bf16-868e162aeec3 is now active with session: c0baf1da-646f-41aa-b1ba-47c71c95becb","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082370688Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082385787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844,"session_id":"c0baf1da-646f-41aa-b1ba-47c71c95becb"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082401786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} {"msg":"Activated sub_volume 2d70bda5-751e-4f52-bf16-868e162aeec3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:08.082415706Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 64 len 36864 data: 14 11 12 13 13 12 11 9 12 02/30 Write block 517 len 28672 data: 17 17 16 16 13 15 13 03/30 Write block 274 len 28672 data: 16 13 13 11 10 8 8 04/30 Flush 05/30 Write block 207 len 40960 data: 16 16 15 13 12 12 9 8 10 10 06/30 Write block 379 len 8192 data: 9 9 07/30 Write block 415 len 32768 data: 9 9 11 11 11 11 12 14 08/30 Read block 392 len 20480 09/30 Read block 432 len 20480 10/30 Write block 516 len 40960 data: 15 18 18 17 17 14 16 14 13 13 11/30 Flush 12/30 Write block 424 len 32768 data: 16 14 14 18 17 14 13 16 13/30 Read block 581 len 24576 14/30 Write block 216 len 12288 data: 11 11 9 15/30 Read block 292 len 36864 16/30 Write block 125 len 28672 data: 14 13 12 13 9 10 10 17/30 Read block 474 len 24576 18/30 Flush 19/30 Write block 470 len 20480 data: 7 9 10 8 8 20/30 Read block 171 len 36864 21/30 Write block 455 len 28672 data: 7 9 8 7 7 8 10 22/30 Read block 31 len 4096 23/30 Read block 86 len 20480 24/30 Read block 12 len 12288 25/30 Read block 66 len 12288 26/30 Flush 27/30 Write block 203 len 8192 data: 14 17 28/30 Read block 83 len 36864 29/30 Read block 159 len 16384 30/30 Write block 259 len 16384 data: 12 10 13 12 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:08.102681684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1844} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 156 156 156 267 267 267 F F F 3 060-079 157 158 158 271 272 272 F F F <--- 4 080-099 154 154 154 263 263 263 F F F 5 100-119 156 156 156 267 267 267 F F F 6 120-139 155 158 158 266 274 274 F F F <--- 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 155 155 155 266 266 266 F F F 10 200-219 156 158 158 269 274 274 F T T <--- 11 220-239 156 156 156 267 267 267 F F F 12 240-259 156 156 156 268 268 268 F T T <--- 13 260-279 154 158 158 263 272 272 F T T <--- 14 280-299 157 158 158 271 272 272 F F F <--- 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 155 158 158 266 273 273 F F F <--- 19 380-399 155 158 158 266 273 273 F F F <--- 20 400-419 152 158 158 258 273 273 F F F <--- 21 420-439 156 158 158 270 274 274 F F F <--- 22 440-459 151 158 158 256 275 275 F F F <--- 23 460-479 156 158 158 267 275 275 F F F <--- 24 480-499 154 154 154 264 264 264 F F F 25 500-519 154 158 158 261 273 273 F F F <--- 26 520-539 156 158 158 267 273 273 F F F <--- 27 540-559 156 156 156 267 267 267 F F F 28 560-579 156 156 156 267 267 267 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 158, Max flush: 275 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 156 156 156 267 267 267 F F F 3 060-079 158 158 158 272 272 272 F F F 4 080-099 154 154 154 263 263 263 F F F 5 100-119 156 156 156 267 267 267 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 155 155 155 266 266 266 F F F 10 200-219 159 159 159 276 276 276 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 159 159 159 276 276 276 F F F 13 260-279 159 159 159 276 276 276 F F F 14 280-299 158 158 158 272 272 272 F F F 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 145 145 145 248 248 248 F F F 18 360-379 158 158 158 273 273 273 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 158 158 158 273 273 273 F F F 21 420-439 158 158 158 274 274 274 F F F 22 440-459 158 158 158 275 275 275 F F F 23 460-479 158 158 158 275 275 275 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 158 158 158 273 273 273 F F F 27 540-559 156 156 156 267 267 267 F F F 28 560-579 156 156 156 267 267 267 F F F 29 580-599 145 145 145 248 248 248 F F F Max gen: 159, Max flush: 276 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.543665534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:12.587317851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.587713886Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.619067272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.619772585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.619818903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.619834352Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.619856362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} {"msg":"Crucible 69a539d8-f9f4-4693-ace8-b629c86483ff has session id: bd77f20e-2862-48bd-af54-b84537b3373f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620482538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Upstairs opts: Upstairs UUID: 69a539d8-f9f4-4693-ace8-b629c86483ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620547405Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Crucible stats registered with UUID: 69a539d8-f9f4-4693-ace8-b629c86483ff","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620574174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:12.620594363Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620770737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620817015Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620840834Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.620978289Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621008238Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} {"msg":"69a539d8-f9f4-4693-ace8-b629c86483ff active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621048566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621084545Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"0","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621124003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"1","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621157042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"2","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621189331Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621226809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621268768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59732)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621307496Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48018)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621343415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46488)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.621372114Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.622667075Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"1","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.622701893Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"0","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.622823878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"2","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.62330495Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623404796Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[0]R flush_numbers[0..12]: [260, 271, 267, 272, 263, 267, 274, 258, 266, 266, 276, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623482553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[0]R generation[0..12]: [153, 157, 156, 158, 154, 156, 158, 152, 155, 155, 159, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623508362Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623528822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[1]R flush_numbers[0..12]: [260, 271, 267, 272, 263, 267, 274, 258, 266, 266, 276, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623547761Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[1]R generation[0..12]: [153, 157, 156, 158, 154, 156, 158, 152, 155, 155, 159, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.62356636Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.62358606Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[2]R flush_numbers[0..12]: [260, 271, 267, 272, 263, 267, 274, 258, 266, 266, 276, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623608568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[2]R generation[0..12]: [153, 157, 156, 158, 154, 156, 158, 152, 155, 155, 159, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623627838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623647407Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623663677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623687085Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623703285Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623719194Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"":"downstairs","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623735984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623751203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"0","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623768042Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"1","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623784832Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"client":"2","":"downstairs","session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623801811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623817811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"69a539d8-f9f4-4693-ace8-b629c86483ff is now active with session: bd77f20e-2862-48bd-af54-b84537b3373f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.62383553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623850919Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852,"session_id":"bd77f20e-2862-48bd-af54-b84537b3373f"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623868669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} {"msg":"Activated sub_volume 69a539d8-f9f4-4693-ace8-b629c86483ff","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:12.623882558Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 85 len 4096 data: 15 02/30 Read block 70 len 32768 03/30 Read block 27 len 12288 04/30 Read block 466 len 4096 05/30 Flush 06/30 Write block 463 len 32768 data: 13 11 12 10 10 9 6 8 07/30 Read block 163 len 32768 08/30 Read block 415 len 32768 09/30 Write block 214 len 4096 data: 9 10/30 Read block 393 len 36864 11/30 Read block 383 len 12288 12/30 Read block 171 len 12288 13/30 Read block 559 len 24576 14/30 Write block 195 len 36864 data: 10 10 12 11 13 16 18 16 15 15/30 Write block 353 len 20480 data: 10 9 8 8 6 16/30 Write block 52 len 36864 data: 14 16 16 11 10 9 10 11 16 17/30 Flush 18/30 Write block 574 len 40960 data: 20 22 20 17 14 13 13 11 10 11 19/30 Read block 428 len 24576 20/30 Read block 7 len 20480 21/30 Read block 50 len 28672 22/30 Read block 23 len 12288 23/30 Read block 297 len 8192 24/30 Read block 180 len 40960 25/30 Write block 559 len 24576 data: 21 17 15 16 16 13 26/30 Read block 291 len 32768 27/30 Read block 321 len 32768 28/30 Write block 242 len 28672 data: 9 9 7 8 10 8 7 29/30 Write block 537 len 32768 data: 14 13 12 9 11 10 10 12 30/30 Read block 277 len 8192 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:12.644460914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1852} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 156 160 160 267 278 278 F F F <--- 3 060-079 158 160 160 272 278 278 F F F <--- 4 080-099 154 160 160 263 277 277 F F F <--- 5 100-119 156 156 156 267 267 267 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 155 160 160 266 278 278 F F F <--- 10 200-219 159 160 160 276 278 278 F F F <--- 11 220-239 156 156 156 267 267 267 F F F 12 240-259 159 159 159 276 276 276 F T T <--- 13 260-279 159 159 159 276 276 276 F F F 14 280-299 158 158 158 272 272 272 F F F 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 145 160 160 248 278 278 F F F <--- 18 360-379 158 158 158 273 273 273 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 158 158 158 273 273 273 F F F 21 420-439 158 158 158 274 274 274 F F F 22 440-459 158 158 158 275 275 275 F F F 23 460-479 158 160 160 275 278 278 F F F <--- 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 158 158 158 273 273 273 F T T <--- 27 540-559 156 156 156 267 267 267 F T T <--- 28 560-579 156 156 156 267 267 267 F T T <--- 29 580-599 145 145 145 248 248 248 F T T <--- Max gen: 160, Max flush: 278 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 260 260 260 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 160 160 160 278 278 278 F F F 3 060-079 160 160 160 278 278 278 F F F 4 080-099 160 160 160 277 277 277 F F F 5 100-119 156 156 156 267 267 267 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 160 160 160 278 278 278 F F F 10 200-219 160 160 160 278 278 278 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 161 161 161 279 279 279 F F F 13 260-279 159 159 159 276 276 276 F F F 14 280-299 158 158 158 272 272 272 F F F 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 158 158 158 273 273 273 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 158 158 158 273 273 273 F F F 21 420-439 158 158 158 274 274 274 F F F 22 440-459 158 158 158 275 275 275 F F F 23 460-479 160 160 160 278 278 278 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 161 161 161 279 279 279 F F F 27 540-559 161 161 161 279 279 279 F F F 28 560-579 161 161 161 279 279 279 F F F 29 580-599 161 161 161 279 279 279 F F F Max gen: 161, Max flush: 279 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.083672461Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.128695106Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.129456707Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.129513955Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.129533364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.129560643Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} {"msg":"Crucible 446ab71e-6446-46fa-9352-8dd40c05ab8f has session id: 88a83e69-f3d1-46a8-a438-3fecab0a20d6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130102602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Upstairs opts: Upstairs UUID: 446ab71e-6446-46fa-9352-8dd40c05ab8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130141571Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Crucible stats registered with UUID: 446ab71e-6446-46fa-9352-8dd40c05ab8f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.13015939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:17.130181069Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130413781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130456669Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130480268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130636982Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130656911Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} {"msg":"446ab71e-6446-46fa-9352-8dd40c05ab8f active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130724889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130745648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"0","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130762787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130779576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"2","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130802056Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130832985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130866903Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39249)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130901342Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41450)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.130933441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:17.131113014Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:17.131160462Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:17.131186751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.131309576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.131352795Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.132332427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"2","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.132367176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"0","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.132954144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:17.133077369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.132173403Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54447)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.13226697Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.13409777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[0]R flush_numbers[0..12]: [260, 271, 278, 278, 277, 267, 274, 258, 266, 278, 278, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134739936Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[0]R generation[0..12]: [153, 157, 160, 160, 160, 156, 158, 152, 155, 160, 160, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134771754Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134797944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[1]R flush_numbers[0..12]: [260, 271, 278, 278, 277, 267, 274, 258, 266, 278, 278, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134821503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[1]R generation[0..12]: [153, 157, 160, 160, 160, 156, 158, 152, 155, 160, 160, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134844852Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134869181Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[2]R flush_numbers[0..12]: [260, 271, 278, 278, 277, 267, 274, 258, 266, 278, 278, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.13489798Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[2]R generation[0..12]: [153, 157, 160, 160, 160, 156, 158, 152, 155, 160, 160, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134921909Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:04:27.134946228Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134970737Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.134991816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135011805Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135031995Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"":"downstairs","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135052984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135071983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"0","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135092832Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"1","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135113501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"client":"2","":"downstairs","session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135134771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-02-15T00:04:27.1351547Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"446ab71e-6446-46fa-9352-8dd40c05ab8f is now active with session: 88a83e69-f3d1-46a8-a438-3fecab0a20d6","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135180409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135199368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860,"session_id":"88a83e69-f3d1-46a8-a438-3fecab0a20d6"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135221997Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} {"msg":"Activated sub_volume 446ab71e-6446-46fa-9352-8dd40c05ab8f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:27.135239617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 253 len 36864 data: 15 11 11 11 10 11 13 11 14 02/30 Write block 74 len 28672 data: 13 10 9 11 12 10 10 03/30 Write block 3 len 24576 data: 8 10 9 8 10 12 04/30 Write block 70 len 40960 data: 12 10 13 14 14 11 10 12 13 11 05/30 Write block 258 len 4096 data: 12 06/30 Read block 468 len 36864 07/30 Read block 132 len 40960 08/30 Write block 402 len 40960 data: 11 15 14 15 16 14 14 14 13 15 09/30 Write block 1 len 36864 data: 9 8 9 11 10 9 11 13 12 10/30 Read block 46 len 36864 11/30 Read block 130 len 24576 12/30 Read block 46 len 8192 13/30 Write block 583 len 32768 data: 12 12 13 13 12 9 8 9 14/30 Read block 116 len 16384 15/30 Read block 288 len 32768 16/30 Flush 17/30 Write block 85 len 28672 data: 16 17 15 15 14 10 9 18/30 Write block 198 len 40960 data: 12 14 17 19 17 16 18 18 15 17 19/30 Write block 279 len 36864 data: 9 9 7 10 10 8 10 10 12 20/30 Write block 576 len 8192 data: 21 18 21/30 Write block 368 len 4096 data: 15 22/30 Write block 423 len 28672 data: 12 17 15 15 19 18 15 23/30 Read block 551 len 8192 24/30 Read block 268 len 40960 25/30 Read block 307 len 20480 26/30 Read block 198 len 24576 27/30 Write block 570 len 32768 data: 15 18 17 20 21 23 22 19 28/30 Read block 289 len 36864 29/30 Read block 409 len 8192 30/30 Read block 580 len 36864 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:27.155377169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1860} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 153 162 280 260 280 F F F <--- 1 020-039 157 157 157 271 271 271 F F F 2 040-059 160 160 160 278 278 278 F F F 3 060-079 162 160 162 280 278 280 F F F <--- 4 080-099 162 160 162 280 277 280 T F T <--- 5 100-119 156 156 156 267 267 267 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 160 160 160 278 278 278 T F T <--- 10 200-219 160 160 160 278 278 278 T F T <--- 11 220-239 156 156 156 267 267 267 F F F 12 240-259 162 161 162 280 279 280 F F F <--- 13 260-279 162 159 162 280 276 280 T F T <--- 14 280-299 158 158 158 272 272 272 T F T <--- 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 158 158 158 273 273 273 T F T <--- 19 380-399 158 158 158 273 273 273 F F F 20 400-419 162 158 162 280 273 280 F F F <--- 21 420-439 158 158 158 274 274 274 T F T <--- 22 440-459 158 158 158 275 275 275 F F F 23 460-479 160 160 160 278 278 278 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 161 161 161 279 279 279 F F F 27 540-559 161 161 161 279 279 279 F F F 28 560-579 161 161 161 279 279 279 T F T <--- 29 580-599 162 161 162 280 279 280 F F F <--- Max gen: 162, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 280 280 280 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 160 160 160 278 278 278 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 163 163 163 281 281 281 F F F 5 100-119 156 156 156 267 267 267 F F F 6 120-139 158 158 158 274 274 274 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 163 163 163 281 281 281 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 163 163 163 281 281 281 F F F 15 300-319 154 154 154 261 261 261 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 158 158 158 273 273 273 F F F 20 400-419 162 162 162 280 280 280 F F F 21 420-439 163 163 163 281 281 281 F F F 22 440-459 158 158 158 275 275 275 F F F 23 460-479 160 160 160 278 278 278 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 161 161 161 279 279 279 F F F 27 540-559 161 161 161 279 279 279 F F F 28 560-579 163 163 163 281 281 281 F F F 29 580-599 162 162 162 280 280 280 F F F Max gen: 163, Max flush: 281 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.597242366Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.644244065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.645462649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.645526106Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.645546386Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.645574025Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} {"msg":"Crucible 3bc99a56-d536-4a8b-8428-35587f997b0f has session id: 5c055cf3-bbed-4986-8f36-7ac266b1b3b4","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.64621192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Upstairs opts: Upstairs UUID: 3bc99a56-d536-4a8b-8428-35587f997b0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646286208Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Crucible stats registered with UUID: 3bc99a56-d536-4a8b-8428-35587f997b0f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646323866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:31.646356125Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646552458Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646595176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646617195Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646815527Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646839016Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} {"msg":"3bc99a56-d536-4a8b-8428-35587f997b0f active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646914164Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646933463Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"0","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646957162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.646987861Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"2","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.64701928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.647051548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.647080137Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45279)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.647114786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38662)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.647142895Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:31.647188243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:31.647227422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:31.64726426Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.647386966Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.647427184Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.648521132Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"0","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.648658197Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"2","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.649225765Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:31.649308162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.648676517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50629)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.648781503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.650282775Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[0]R flush_numbers[0..12]: [280, 271, 278, 280, 281, 267, 274, 258, 266, 281, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.650935591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[0]R generation[0..12]: [162, 157, 160, 162, 163, 156, 158, 152, 155, 163, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.650972679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.650998648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[1]R flush_numbers[0..12]: [280, 271, 278, 280, 281, 267, 274, 258, 266, 281, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651022507Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[1]R generation[0..12]: [162, 157, 160, 162, 163, 156, 158, 152, 155, 163, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651047516Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651077755Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[2]R flush_numbers[0..12]: [280, 271, 278, 280, 281, 267, 274, 258, 266, 281, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651101494Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[2]R generation[0..12]: [162, 157, 160, 162, 163, 156, 158, 152, 155, 163, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651125123Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651149413Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651169822Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651191111Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:04:41.65121102Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651234239Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"":"downstairs","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651255729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651274898Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"0","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651296637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"1","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651313436Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"client":"2","":"downstairs","session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651330526Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651346515Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"3bc99a56-d536-4a8b-8428-35587f997b0f is now active with session: 5c055cf3-bbed-4986-8f36-7ac266b1b3b4","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651364294Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651381684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868,"session_id":"5c055cf3-bbed-4986-8f36-7ac266b1b3b4"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651398073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} {"msg":"Activated sub_volume 3bc99a56-d536-4a8b-8428-35587f997b0f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:41.651412092Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 380 len 12288 data: 10 10 9 02/30 Write block 46 len 28672 data: 11 15 16 16 14 14 15 03/30 Write block 532 len 36864 data: 11 11 13 13 14 15 14 13 10 04/30 Write block 574 len 28672 data: 22 24 23 20 15 14 14 05/30 Read block 81 len 12288 06/30 Write block 423 len 36864 data: 13 18 16 16 20 19 16 14 17 07/30 Read block 69 len 28672 08/30 Read block 438 len 12288 09/30 Write block 563 len 20480 data: 17 14 9 7 8 10/30 Read block 436 len 20480 11/30 Read block 301 len 24576 12/30 Read block 499 len 4096 13/30 Read block 160 len 36864 14/30 Read block 94 len 28672 15/30 Read block 394 len 8192 16/30 Flush 17/30 Write block 310 len 32768 data: 13 16 18 17 15 13 11 13 18/30 Read block 478 len 20480 19/30 Flush 20/30 Write block 98 len 40960 data: 10 10 15 16 16 16 14 12 11 11 21/30 Write block 128 len 8192 data: 14 10 22/30 Read block 181 len 12288 23/30 Write block 18 len 8192 data: 12 13 24/30 Write block 445 len 8192 data: 13 13 25/30 Flush 26/30 Write block 450 len 12288 data: 8 10 8 27/30 Read block 128 len 32768 28/30 Read block 547 len 16384 29/30 Write block 461 len 24576 data: 11 13 14 12 13 11 30/30 Read block 429 len 20480 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:41.67169212Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1868} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 162 164 284 280 284 F F F <--- 1 020-039 157 157 157 271 271 271 F F F 2 040-059 164 160 164 282 278 282 F T F <--- 3 060-079 162 162 162 280 280 280 F F F 4 080-099 164 163 164 284 281 284 F F F <--- 5 100-119 164 156 164 284 267 284 F F F <--- 6 120-139 164 158 164 284 274 284 F F F <--- 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 163 163 163 281 281 281 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 163 163 163 281 281 281 F F F 15 300-319 164 154 164 283 261 283 F F F <--- 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 164 158 164 282 273 282 F T F <--- 20 400-419 162 162 162 280 280 280 F F F 21 420-439 164 163 164 282 281 282 F F F <--- 22 440-459 164 158 164 284 275 284 T F T <--- 23 460-479 160 160 160 278 278 278 T F T <--- 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 164 161 164 282 279 282 F F F <--- 27 540-559 164 161 164 282 279 282 F F F <--- 28 560-579 164 163 164 282 281 282 F F F <--- 29 580-599 164 162 164 282 280 282 F F F <--- Max gen: 164, Max flush: 284 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 164 164 164 282 282 282 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 164 164 164 284 284 284 F F F 5 100-119 164 164 164 284 284 284 F F F 6 120-139 164 164 164 284 284 284 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 155 155 155 266 266 266 F F F 9 180-199 163 163 163 281 281 281 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 163 163 163 281 281 281 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 164 164 164 282 282 282 F F F 20 400-419 162 162 162 280 280 280 F F F 21 420-439 164 164 164 282 282 282 F F F 22 440-459 165 165 165 285 285 285 F F F 23 460-479 165 165 165 285 285 285 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 164 164 164 282 282 282 F F F 27 540-559 164 164 164 282 282 282 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 164 164 164 282 282 282 F F F Max gen: 165, Max flush: 285 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.09874242Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.146892466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.14781938Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.147870819Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.147887838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.147910267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} {"msg":"Crucible 54d076bc-28f0-42d9-87b6-a4cd17738657 has session id: 461788cb-146d-4f29-8bb0-fced76a4369c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.148407928Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Upstairs opts: Upstairs UUID: 54d076bc-28f0-42d9-87b6-a4cd17738657, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.148464836Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Crucible stats registered with UUID: 54d076bc-28f0-42d9-87b6-a4cd17738657","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.148486665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:46.148503274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.148755135Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.148807003Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.148843211Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149124521Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149171899Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} {"msg":"54d076bc-28f0-42d9-87b6-a4cd17738657 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149210437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149240286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"0","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149275045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"1","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149307334Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"2","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149338723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149370491Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.14940176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35809)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149498337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55200)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149542145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39654)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.149579563Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.150873884Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"0","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.151056377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"2","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.151639665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.151730612Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"1","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.15177368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[0]R flush_numbers[0..12]: [284, 271, 282, 280, 284, 284, 284, 258, 266, 281, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152466143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[0]R generation[0..12]: [164, 157, 164, 162, 164, 164, 164, 152, 155, 163, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152507192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152531361Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[1]R flush_numbers[0..12]: [284, 271, 282, 280, 284, 284, 284, 258, 266, 281, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.15255633Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[1]R generation[0..12]: [164, 157, 164, 162, 164, 164, 164, 152, 155, 163, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152588079Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152627677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[2]R flush_numbers[0..12]: [284, 271, 282, 280, 284, 284, 284, 258, 266, 281, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152663596Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[2]R generation[0..12]: [164, 157, 164, 162, 164, 164, 164, 152, 155, 163, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152701304Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152730273Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152752293Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"Wait for a query_work_queue command to finish before sending IO } {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152788241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.15282031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152843719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"":"downstairs","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152863798Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152890827Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"0","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152923406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"1","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152945905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"client":"2","":"downstairs","session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152963165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.152990153Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"54d076bc-28f0-42d9-87b6-a4cd17738657 is now active with session: 461788cb-146d-4f29-8bb0-fced76a4369c","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.153022122Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.153052151Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876,"session_id":"461788cb-146d-4f29-8bb0-fced76a4369c"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.15307208Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} {"msg":"Activated sub_volume 54d076bc-28f0-42d9-87b6-a4cd17738657","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:46.15309001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 4096 data: 9 02/30 Read block 498 len 32768 03/30 Write block 423 len 24576 data: 14 19 17 17 21 20 04/30 Read block 569 len 4096 05/30 Read block 429 len 32768 06/30 Read block 446 len 40960 07/30 Read block 474 len 36864 08/30 Write block 290 len 16384 data: 11 12 13 14 09/30 Write block 391 len 12288 data: 13 13 16 10/30 Read block 248 len 24576 11/30 Read block 98 len 40960 12/30 Read block 313 len 36864 13/30 Read block 193 len 20480 14/30 Write block 434 len 28672 data: 15 16 15 15 16 17 16 15/30 Write block 162 len 16384 data: 13 12 10 10 16/30 Write block 539 len 40960 data: 14 11 12 11 11 13 10 11 11 13 17/30 Read block 474 len 4096 18/30 Read block 258 len 36864 19/30 Read block 293 len 8192 20/30 Write block 304 len 32768 data: 15 17 16 16 14 17 14 17 21/30 Read block 214 len 24576 22/30 Write block 190 len 28672 data: 10 11 9 8 9 11 11 23/30 Read block 42 len 16384 24/30 Flush 25/30 Write block 162 len 8192 data: 14 13 26/30 Write block 581 len 8192 data: 12 11 27/30 Read block 550 len 32768 28/30 Read block 442 len 16384 29/30 Read block 265 len 24576 30/30 Read block 378 len 28672 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:46.17617417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1876} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 164 164 164 282 282 282 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 164 164 164 284 284 284 F F F 5 100-119 166 164 166 286 284 286 F T F <--- 6 120-139 164 164 164 284 284 284 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 166 155 166 286 266 286 T F T <--- 9 180-199 166 163 166 286 281 286 F F F <--- 10 200-219 163 163 163 281 281 281 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 166 163 166 286 281 286 F F F <--- 15 300-319 166 164 166 286 283 286 F F F <--- 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 164 166 286 282 286 F F F <--- 20 400-419 162 162 162 280 280 280 F F F 21 420-439 166 164 166 286 282 286 F T F <--- 22 440-459 166 165 166 286 285 286 F F F <--- 23 460-479 165 165 165 285 285 285 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 166 164 166 286 282 286 F F F <--- 27 540-559 166 164 166 286 282 286 F F F <--- 28 560-579 164 164 164 282 282 282 F F F 29 580-599 164 164 164 282 282 282 T F T <--- Max gen: 166, Max flush: 286 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 157 157 157 271 271 271 F F F 2 040-059 164 164 164 282 282 282 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 164 164 164 284 284 284 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 164 164 164 284 284 284 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 167 167 167 287 287 287 F F F 9 180-199 166 166 166 286 286 286 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 156 156 156 267 267 267 F F F 12 240-259 162 162 162 280 280 280 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 166 166 166 286 286 286 F F F 15 300-319 166 166 166 286 286 286 F F F 16 320-339 157 157 157 271 271 271 F F F 17 340-359 160 160 160 278 278 278 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 162 162 162 280 280 280 F F F 21 420-439 166 166 166 286 286 286 F F F 22 440-459 166 166 166 286 286 286 F F F 23 460-479 165 165 165 285 285 285 F F F 24 480-499 154 154 154 264 264 264 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 166 166 166 286 286 286 F F F 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 167 167 167 287 287 287 F F F Max gen: 167, Max flush: 287 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.581012057Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.626378139Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627281785Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627332973Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627349442Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627376741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} {"msg":"Crucible 1a014ac4-34ea-45ff-948f-a8f7c53936f9 has session id: 88b2f67c-6525-44f3-9a49-217008e0a160","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627905311Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Upstairs opts: Upstairs UUID: 1a014ac4-34ea-45ff-948f-a8f7c53936f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627959329Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Crucible stats registered with UUID: 1a014ac4-34ea-45ff-948f-a8f7c53936f9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.627990138Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:50.628017207Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628244578Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628297546Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628328625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628498918Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628528807Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} {"msg":"1a014ac4-34ea-45ff-948f-a8f7c53936f9 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628586265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628621234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"0","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628654742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"1","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628691521Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"2","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628729019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628765168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628804317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"ds_connection connected from Ok(127.0.0.1:41556)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628836806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"ds_connection connected from Ok(127.0.0.1:41356)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.628867434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"ds_connection connected from Ok(127.0.0.1:41128)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.629084606Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.629964312Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"0","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.630055349Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"2","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.630541001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.630660356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.631793853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"1","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[0]R flush_numbers[0..12]: [284, 271, 282, 280, 284, 286, 284, 258, 287, 286, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632347762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[0]R generation[0..12]: [164, 157, 164, 162, 164, 166, 164, 152, 167, 166, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632377911Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.63239917Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[1]R flush_numbers[0..12]: [284, 271, 282, 280, 284, 286, 284, 258, 287, 286, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632426049Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[1]R generation[0..12]: [164, 157, 164, 162, 164, 166, 164, 152, 167, 166, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632448538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632468577Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[2]R flush_numbers[0..12]: [284, 271, 282, 280, 284, 286, 284, 258, 287, 286, 281, 267]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632487676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[2]R generation[0..12]: [164, 157, 164, 162, 164, 166, 164, 152, 167, 166, 163, 156]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632506566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632526045Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Max found gen is 168"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632549764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632567073Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632583103Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632599212Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"":"downstairs","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632616342Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632631661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"0","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.6326485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"1","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.63266534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"client":"2","":"downstairs","session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632682509Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632698608Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"1a014ac4-34ea-45ff-948f-a8f7c53936f9 is now active with session: 88b2f67c-6525-44f3-9a49-217008e0a160","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632718327Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632737517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884,"session_id":"88b2f67c-6525-44f3-9a49-217008e0a160"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632753376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} {"msg":"Activated sub_volume 1a014ac4-34ea-45ff-948f-a8f7c53936f9","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:50.632767336Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 224 len 12288 data: 13 9 6 02/30 Read block 258 len 4096 03/30 Read block 428 len 36864 04/30 Read block 26 len 36864 05/30 Write block 471 len 16384 data: 10 11 9 9 06/30 Read block 53 len 4096 07/30 Read block 82 len 36864 08/30 Write block 289 len 16384 data: 12 12 13 14 09/30 Read block 520 len 24576 10/30 Write block 25 len 28672 data: 15 14 13 16 12 11 10 11/30 Write block 251 len 20480 data: 13 13 16 12 12 12/30 Read block 29 len 4096 13/30 Write block 583 len 28672 data: 13 13 14 14 13 10 9 14/30 Write block 461 len 4096 data: 12 15/30 Read block 78 len 28672 16/30 Write block 49 len 16384 data: 17 15 15 16 17/30 Read block 565 len 12288 18/30 Read block 131 len 8192 19/30 Write block 342 len 4096 data: 16 20/30 Read block 188 len 32768 21/30 Write block 322 len 32768 data: 13 11 7 9 9 8 9 9 22/30 Flush 23/30 Write block 298 len 20480 data: 17 17 15 17 19 24/30 Read block 506 len 4096 25/30 Read block 147 len 32768 26/30 Write block 481 len 36864 data: 11 12 16 16 17 16 19 19 18 27/30 Write block 83 len 12288 data: 13 14 17 28/30 Read block 252 len 24576 29/30 Read block 256 len 32768 30/30 Read block 303 len 12288 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:50.652101889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1884} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 168 157 168 288 271 288 F F F <--- 2 040-059 168 164 168 288 282 288 F F F <--- 3 060-079 162 162 162 280 280 280 F F F 4 080-099 164 164 164 284 284 284 T F T <--- 5 100-119 166 166 166 286 286 286 F F F 6 120-139 164 164 164 284 284 284 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 167 167 167 287 287 287 F F F 9 180-199 166 166 166 286 286 286 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 168 156 168 288 267 288 F F F <--- 12 240-259 168 162 168 288 280 288 F F F <--- 13 260-279 163 163 163 281 281 281 F F F 14 280-299 168 166 168 288 286 288 T F T <--- 15 300-319 166 166 166 286 286 286 T F T <--- 16 320-339 168 157 168 288 271 288 F F F <--- 17 340-359 168 160 168 288 278 288 F F F <--- 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 162 162 162 280 280 280 F F F 21 420-439 166 166 166 286 286 286 F F F 22 440-459 166 166 166 286 286 286 F F F 23 460-479 168 165 168 288 285 288 F F F <--- 24 480-499 154 154 154 264 264 264 T F T <--- 25 500-519 158 158 158 273 273 273 F F F 26 520-539 166 166 166 286 286 286 F F F 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 168 167 168 288 287 288 F F F <--- Max gen: 168, Max flush: 288 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 168 168 168 288 288 288 F F F 2 040-059 168 168 168 288 288 288 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 164 164 164 284 284 284 F F F 7 140-159 152 152 152 258 258 258 F F F 8 160-179 167 167 167 287 287 287 F F F 9 180-199 166 166 166 286 286 286 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 168 168 168 288 288 288 F F F 12 240-259 168 168 168 288 288 288 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 169 169 169 289 289 289 F F F 16 320-339 168 168 168 288 288 288 F F F 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 162 162 162 280 280 280 F F F 21 420-439 166 166 166 286 286 286 F F F 22 440-459 166 166 166 286 286 286 F F F 23 460-479 168 168 168 288 288 288 F F F 24 480-499 169 169 169 289 289 289 F F F 25 500-519 158 158 158 273 273 273 F F F 26 520-539 166 166 166 286 286 286 F F F 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 168 168 168 288 288 288 F F F Max gen: 169, Max flush: 289 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.06073391Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:55.103859257Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.10430317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.135255511Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.135912166Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.135959794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.135975243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.135997053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} {"msg":"Crucible 736c1873-6e55-403d-9756-55ac30edf55b has session id: 14f1d672-6b1e-465d-911f-e0369b1027c5","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.136432796Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Upstairs opts: Upstairs UUID: 736c1873-6e55-403d-9756-55ac30edf55b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.136551602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Crucible stats registered with UUID: 736c1873-6e55-403d-9756-55ac30edf55b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.13658465Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:55.13660739Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.136795922Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.13684429Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.136874479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137055093Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137084731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} {"msg":"736c1873-6e55-403d-9756-55ac30edf55b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137144959Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137165978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"0","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137183428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"1","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137200187Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"2","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137216556Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137239495Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137270094Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38086)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137347531Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35119)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.13738977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51998)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.137422028Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.138735818Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"1","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.138839874Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"2","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.138873853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"0","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139361094Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139390584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[0]R flush_numbers[0..12]: [284, 288, 288, 280, 289, 286, 284, 258, 287, 286, 281, 288]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139409113Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[0]R generation[0..12]: [164, 168, 168, 162, 169, 166, 164, 152, 167, 166, 163, 168]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139428582Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139448141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[1]R flush_numbers[0..12]: [284, 288, 288, 280, 289, 286, 284, 258, 287, 286, 281, 288]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.13946717Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[1]R generation[0..12]: [164, 168, 168, 162, 169, 166, 164, 152, 167, 166, 163, 168]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.13948647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139506689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[2]R flush_numbers[0..12]: [284, 288, 288, 280, 289, 286, 284, 258, 287, 286, 281, 288]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139525628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[2]R generation[0..12]: [164, 168, 168, 162, 169, 166, 164, 152, 167, 166, 163, 168]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139544408Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139563917Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139583766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139600945Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139617215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139633234Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"":"downstairs","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139650394Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139666993Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"0","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:04:55.139684142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"1","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139709421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"client":"2","":"downstairs","session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139726501Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.13974256Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"736c1873-6e55-403d-9756-55ac30edf55b is now active with session: 14f1d672-6b1e-465d-911f-e0369b1027c5","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139760499Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139775539Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892,"session_id":"14f1d672-6b1e-465d-911f-e0369b1027c5"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139791488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} {"msg":"Activated sub_volume 736c1873-6e55-403d-9756-55ac30edf55b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:55.139805398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 229 len 20480 data: 9 12 13 13 11 02/30 Read block 433 len 4096 03/30 Flush 04/30 Write block 445 len 20480 data: 14 14 12 11 9 05/30 Write block 507 len 8192 data: 8 8 06/30 Write block 534 len 4096 data: 14 07/30 Read block 454 len 12288 08/30 Read block 18 len 20480 09/30 Write block 191 len 8192 data: 12 10 10/30 Read block 159 len 20480 11/30 Read block 130 len 16384 12/30 Write block 447 len 36864 data: 13 12 10 9 11 9 9 8 8 13/30 Flush 14/30 Write block 449 len 8192 data: 11 10 15/30 Write block 328 len 32768 data: 10 10 10 10 9 10 9 9 16/30 Read block 28 len 12288 17/30 Flush 18/30 Write block 314 len 16384 data: 16 14 12 14 19/30 Read block 209 len 16384 20/30 Read block 248 len 24576 21/30 Write block 133 len 4096 data: 9 22/30 Read block 535 len 24576 23/30 Write block 169 len 28672 data: 8 12 12 14 14 13 13 24/30 Write block 411 len 16384 data: 16 14 13 13 25/30 Write block 159 len 16384 data: 15 13 14 15 26/30 Write block 422 len 4096 data: 15 27/30 Write block 457 len 28672 data: 9 8 8 9 13 14 15 28/30 Read block 513 len 8192 29/30 Read block 229 len 16384 30/30 Read block 36 len 36864 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 0 26 27 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:55.157694466Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1892} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 168 168 168 288 288 288 F F F 2 040-059 168 168 168 288 288 288 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 164 164 164 284 284 284 F T T <--- 7 140-159 152 152 152 258 258 258 F T T <--- 8 160-179 167 167 167 287 287 287 F T T <--- 9 180-199 166 170 170 286 291 291 F F F <--- 10 200-219 163 163 163 281 281 281 F F F 11 220-239 170 170 170 290 290 290 F F F 12 240-259 168 168 168 288 288 288 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 169 169 169 289 289 289 F T T <--- 16 320-339 168 170 170 288 292 292 F F F <--- 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 162 162 162 280 280 280 F T T <--- 21 420-439 166 166 166 286 286 286 F T T <--- 22 440-459 166 170 170 286 292 292 F T T <--- 23 460-479 168 168 168 288 288 288 F T T <--- 24 480-499 169 169 169 289 289 289 F F F 25 500-519 158 170 170 273 291 291 F F F <--- 26 520-539 166 170 170 286 291 291 F F F <--- 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 168 168 168 288 288 288 F F F Max gen: 170, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 168 168 168 288 288 288 F F F 2 040-059 168 168 168 288 288 288 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 171 171 171 293 293 293 F F F 7 140-159 171 171 171 293 293 293 F F F 8 160-179 171 171 171 293 293 293 F F F 9 180-199 170 170 170 291 291 291 F F F 10 200-219 163 163 163 281 281 281 F F F 11 220-239 170 170 170 290 290 290 F F F 12 240-259 168 168 168 288 288 288 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 171 171 171 293 293 293 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 171 171 171 293 293 293 F F F 22 440-459 171 171 171 293 293 293 F F F 23 460-479 171 171 171 293 293 293 F F F 24 480-499 169 169 169 289 289 289 F F F 25 500-519 170 170 170 291 291 291 F F F 26 520-539 170 170 170 291 291 291 F F F 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 168 168 168 288 288 288 F F F Max gen: 171, Max flush: 293 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.605057272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.650212422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651104908Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651153376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651174625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651197105Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} {"msg":"Crucible 019b3997-6e07-41d3-87ed-71d1898f47fa has session id: 5f56789e-7bd2-4c4e-b569-0a9e3151c93f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651731384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Upstairs opts: Upstairs UUID: 019b3997-6e07-41d3-87ed-71d1898f47fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651787712Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Crucible stats registered with UUID: 019b3997-6e07-41d3-87ed-71d1898f47fa","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651817741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:59.65184396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.651993764Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652034523Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652054422Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652192407Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652215866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} {"msg":"019b3997-6e07-41d3-87ed-71d1898f47fa active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652291913Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652311722Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"0","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652332171Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"1","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652349371Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"2","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.65236647Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652394559Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652428158Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48270)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652461176Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47802)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652491695Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49180)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.652670679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.653831914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"2","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.653877202Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"0","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.654499599Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.654530398Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.655916365Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"1","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[0]R flush_numbers[0..12]: [284, 288, 288, 280, 289, 286, 293, 293, 293, 291, 281, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656477074Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[0]R generation[0..12]: [164, 168, 168, 162, 169, 166, 171, 171, 171, 170, 163, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656505493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656530352Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[1]R flush_numbers[0..12]: [284, 288, 288, 280, 289, 286, 293, 293, 293, 291, 281, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656549821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[1]R generation[0..12]: [164, 168, 168, 162, 169, 166, 171, 171, 171, 170, 163, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.65656889Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656588359Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[2]R flush_numbers[0..12]: [284, 288, 288, 280, 289, 286, 293, 293, 293, 291, 281, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656607309Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[2]R generation[0..12]: [164, 168, 168, 162, 169, 166, 171, 171, 171, 170, 163, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656626168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656645607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656661747Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656678736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656694605Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:04:59.656710675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"":"downstairs","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656732884Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656748203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"0","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656768132Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"1","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656787812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"client":"2","":"downstairs","session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656810091Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.65682923Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"019b3997-6e07-41d3-87ed-71d1898f47fa is now active with session: 5f56789e-7bd2-4c4e-b569-0a9e3151c93f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656850269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656868019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900,"session_id":"5f56789e-7bd2-4c4e-b569-0a9e3151c93f"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656887828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} {"msg":"Activated sub_volume 019b3997-6e07-41d3-87ed-71d1898f47fa","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:04:59.656904387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 127 len 8192 data: 13 15 03/30 Read block 46 len 32768 04/30 Flush 05/30 Write block 205 len 12288 data: 19 16 18 06/30 Read block 360 len 32768 07/30 Flush 08/30 Write block 194 len 20480 data: 10 12 12 13 13 09/30 Write block 491 len 40960 data: 17 15 13 11 11 13 10 10 9 11 10/30 Read block 402 len 20480 11/30 Read block 513 len 40960 12/30 Read block 79 len 28672 13/30 Write block 488 len 32768 data: 20 19 19 18 16 14 12 12 14/30 Read block 422 len 32768 15/30 Read block 404 len 4096 16/30 Read block 50 len 16384 17/30 Write block 5 len 20480 data: 11 10 12 14 13 18/30 Read block 204 len 12288 19/30 Read block 166 len 28672 20/30 Read block 217 len 8192 21/30 Flush 22/30 Write block 2 len 32768 data: 9 10 12 12 11 13 15 14 23/30 Read block 212 len 12288 24/30 Read block 253 len 20480 25/30 Write block 174 len 40960 data: 14 14 13 11 12 11 7 5 6 5 26/30 Write block 461 len 40960 data: 14 15 16 13 14 12 11 10 7 9 27/30 Read block 205 len 28672 28/30 Write block 514 len 16384 data: 8 10 16 19 29/30 Read block 161 len 8192 30/30 Write block 523 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:04:59.677966255Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1900} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 164 172 297 284 297 T F T <--- 1 020-039 168 168 168 288 288 288 F F F 2 040-059 168 168 168 288 288 288 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 172 171 172 295 293 295 F F F <--- 7 140-159 171 171 171 293 293 293 F F F 8 160-179 171 171 171 293 293 293 T F T <--- 9 180-199 172 170 172 297 291 297 T F T <--- 10 200-219 172 163 172 296 281 296 F F F <--- 11 220-239 170 170 170 290 290 290 F F F 12 240-259 168 168 168 288 288 288 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 171 171 171 293 293 293 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 171 171 171 293 293 293 F F F 22 440-459 171 171 171 293 293 293 F F F 23 460-479 171 171 171 293 293 293 T F T <--- 24 480-499 172 169 172 297 289 297 F F F <--- 25 500-519 172 170 172 297 291 297 T F T <--- 26 520-539 170 170 170 291 291 291 T F T <--- 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 168 168 168 288 288 288 F F F Max gen: 172, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 298 298 298 F F F 1 020-039 168 168 168 288 288 288 F F F 2 040-059 168 168 168 288 288 288 F F F 3 060-079 162 162 162 280 280 280 F F F 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 172 172 172 295 295 295 F F F 7 140-159 171 171 171 293 293 293 F F F 8 160-179 173 173 173 298 298 298 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 172 172 172 296 296 296 F F F 11 220-239 170 170 170 290 290 290 F F F 12 240-259 168 168 168 288 288 288 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 171 171 171 293 293 293 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 166 166 286 286 286 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 171 171 171 293 293 293 F F F 22 440-459 171 171 171 293 293 293 F F F 23 460-479 173 173 173 298 298 298 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 173 173 173 298 298 298 F F F 27 540-559 166 166 166 286 286 286 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 168 168 168 288 288 288 F F F Max gen: 173, Max flush: 298 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.085324975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:04.129603778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.129969114Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.161039811Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.161722545Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.161775803Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.161795622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.161824221Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} {"msg":"Crucible 6706a77d-6968-40bb-bf34-227d9a97552b has session id: dfec9080-39ea-473d-860d-cabcb95af605","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162205806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Upstairs opts: Upstairs UUID: 6706a77d-6968-40bb-bf34-227d9a97552b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162278524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Crucible stats registered with UUID: 6706a77d-6968-40bb-bf34-227d9a97552b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162318412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:04.162353231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162508965Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162551953Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162572372Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162710937Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162730766Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} {"msg":"6706a77d-6968-40bb-bf34-227d9a97552b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162793214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162813043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"0","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162830093Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"1","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162846792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"2","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162870611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.16290001Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162938389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"ds_connection connected from Ok(127.0.0.1:40173)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.162964637Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"ds_connection connected from Ok(127.0.0.1:36310)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.163033195Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"ds_connection connected from Ok(127.0.0.1:50457)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.163073333Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.164411222Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"1","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.164445171Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"2","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.16447607Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"0","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165023929Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165132335Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[0]R flush_numbers[0..12]: [298, 288, 288, 280, 289, 286, 295, 293, 298, 298, 296, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165163134Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[0]R generation[0..12]: [173, 168, 168, 162, 169, 166, 172, 171, 173, 173, 172, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165183173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165204102Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[1]R flush_numbers[0..12]: [298, 288, 288, 280, 289, 286, 295, 293, 298, 298, 296, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165223522Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[1]R generation[0..12]: [173, 168, 168, 162, 169, 166, 172, 171, 173, 173, 172, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165242451Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.16526215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[2]R flush_numbers[0..12]: [298, 288, 288, 280, 289, 286, 295, 293, 298, 298, 296, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165280999Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[2]R generation[0..12]: [173, 168, 168, 162, 169, 166, 172, 171, 173, 173, 172, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165304358Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165324507Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165340887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165357736Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165373686Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165389825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"":"downstairs","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165406675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165421854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"0","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:05:04.165438603Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"1","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165462842Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"client":"2","":"downstairs","session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165480072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165496061Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"6706a77d-6968-40bb-bf34-227d9a97552b is now active with session: dfec9080-39ea-473d-860d-cabcb95af605","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.16551385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.16552904Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908,"session_id":"dfec9080-39ea-473d-860d-cabcb95af605"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165545239Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} {"msg":"Activated sub_volume 6706a77d-6968-40bb-bf34-227d9a97552b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:04.165559059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 126 len 12288 data: 14 14 16 02/30 Read block 426 len 24576 03/30 Read block 426 len 8192 04/30 Read block 518 len 8192 05/30 Write block 553 len 36864 data: 14 15 13 15 18 18 22 18 16 06/30 Read block 495 len 32768 07/30 Write block 420 len 8192 data: 12 13 08/30 Write block 547 len 20480 data: 12 14 12 13 13 09/30 Flush 10/30 Write block 586 len 4096 data: 15 11/30 Read block 111 len 40960 12/30 Read block 307 len 36864 13/30 Read block 418 len 16384 14/30 Write block 216 len 12288 data: 12 12 10 15/30 Read block 310 len 20480 16/30 Read block 200 len 40960 17/30 Flush 18/30 Write block 385 len 8192 data: 10 9 19/30 Read block 138 len 40960 20/30 Write block 57 len 36864 data: 10 11 12 17 13 15 13 15 12 21/30 Flush 22/30 Write block 300 len 16384 data: 16 18 20 18 23/30 Read block 187 len 28672 24/30 Read block 481 len 16384 25/30 Read block 317 len 8192 26/30 Read block 333 len 4096 27/30 Write block 249 len 24576 data: 9 12 14 14 17 13 28/30 Write block 18 len 8192 data: 13 14 29/30 Read block 480 len 20480 30/30 Write block 21 len 28672 data: 19 15 14 17 16 15 14 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 1 1 26 Done 6 29 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:04.185027017Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1908} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 298 298 298 F T T <--- 1 020-039 168 168 168 288 288 288 F T T <--- 2 040-059 168 174 174 288 301 301 F F F <--- 3 060-079 162 174 174 280 301 301 F F F <--- 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 172 174 174 295 299 299 T F F <--- 7 140-159 171 171 171 293 293 293 F F F 8 160-179 173 173 173 298 298 298 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 172 174 174 296 300 300 F F F <--- 11 220-239 170 170 170 290 290 290 F F F 12 240-259 168 168 168 288 288 288 F T T <--- 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 171 171 171 293 293 293 F T T <--- 16 320-339 170 170 170 292 292 292 F F F 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 166 174 174 286 301 301 F F F <--- 20 400-419 171 171 171 293 293 293 F F F 21 420-439 171 174 174 293 299 299 F F F <--- 22 440-459 171 171 171 293 293 293 F F F 23 460-479 173 173 173 298 298 298 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 173 173 173 298 298 298 F F F 27 540-559 166 174 174 286 299 299 T F F <--- 28 560-579 164 174 174 282 299 299 T F F <--- 29 580-599 168 174 174 288 300 300 F F F <--- Max gen: 174, Max flush: 301 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 302 302 302 F F F 1 020-039 175 175 175 302 302 302 F F F 2 040-059 174 174 174 301 301 301 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 169 169 169 289 289 289 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 171 171 171 293 293 293 F F F 8 160-179 173 173 173 298 298 298 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 174 174 174 300 300 300 F F F 11 220-239 170 170 170 290 290 290 F F F 12 240-259 175 175 175 302 302 302 F F F 13 260-279 163 163 163 281 281 281 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 168 168 168 288 288 288 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 174 174 174 301 301 301 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 174 174 174 299 299 299 F F F 22 440-459 171 171 171 293 293 293 F F F 23 460-479 173 173 173 298 298 298 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 173 173 173 298 298 298 F F F 27 540-559 174 174 174 299 299 299 F F F 28 560-579 174 174 174 299 299 299 F F F 29 580-599 174 174 174 300 300 300 F F F Max gen: 175, Max flush: 302 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.624513524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.670314259Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671004213Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671056441Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671073611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.67109837Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} {"msg":"Crucible d06ce273-d7e5-423b-97e2-84ff07caacd7 has session id: 6036b1f7-c9ff-46ea-993b-c296836c358f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671526673Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Upstairs opts: Upstairs UUID: d06ce273-d7e5-423b-97e2-84ff07caacd7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671643939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Crucible stats registered with UUID: d06ce273-d7e5-423b-97e2-84ff07caacd7","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671678738Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:08.671701887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671917678Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671973856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.671997395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672189978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672219977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} {"msg":"d06ce273-d7e5-423b-97e2-84ff07caacd7 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672299714Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672325223Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"0","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672346772Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672377971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"2","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.67241661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672453468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672492356Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:08.672546685Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:08.672589243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:08.672630121Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50686)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.67266285Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53588)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672754367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672796985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.672827704Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.673744469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"0","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.673778687Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"2","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.674309897Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:08.674389154Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.6742196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44527)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.674331646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.675826679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[0]R flush_numbers[0..12]: [302, 302, 301, 301, 289, 286, 299, 293, 298, 298, 300, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676529202Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[0]R generation[0..12]: [175, 175, 174, 174, 169, 166, 174, 171, 173, 173, 174, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.67656665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.67659151Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[1]R flush_numbers[0..12]: [302, 302, 301, 301, 289, 286, 299, 293, 298, 298, 300, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676613449Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[1]R generation[0..12]: [175, 175, 174, 174, 169, 166, 174, 171, 173, 173, 174, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676646467Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676669357Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[2]R flush_numbers[0..12]: [302, 302, 301, 301, 289, 286, 299, 293, 298, 298, 300, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676691016Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[2]R generation[0..12]: [175, 175, 174, 174, 169, 166, 174, 171, 173, 173, 174, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676716165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676742174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:05:18.676774863Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676808661Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.67683018Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676860579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"":"downstairs","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676893788Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676924077Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"0","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.676958656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"1","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2026-02-15T00:05:18.676992304Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"client":"2","":"downstairs","session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.677027243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.677051952Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"d06ce273-d7e5-423b-97e2-84ff07caacd7 is now active with session: 6036b1f7-c9ff-46ea-993b-c296836c358f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.677072711Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.677089931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916,"session_id":"6036b1f7-c9ff-46ea-993b-c296836c358f"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.67710808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} {"msg":"Activated sub_volume d06ce273-d7e5-423b-97e2-84ff07caacd7","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:18.677123799Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 20480 data: 15 16 17 14 15 02/30 Read block 157 len 20480 03/30 Read block 377 len 36864 04/30 Write block 151 len 28672 data: 14 15 16 13 14 13 12 05/30 Write block 214 len 24576 data: 10 11 13 13 11 7 06/30 Write block 83 len 16384 data: 14 15 18 18 07/30 Read block 125 len 32768 08/30 Write block 271 len 24576 data: 13 13 13 17 14 14 09/30 Read block 328 len 4096 10/30 Read block 394 len 28672 11/30 Read block 343 len 8192 12/30 Write block 173 len 16384 data: 15 15 15 14 13/30 Read block 169 len 36864 14/30 Write block 441 len 36864 data: 14 11 11 11 15 15 14 13 12 15/30 Read block 282 len 8192 16/30 Write block 592 len 16384 data: 9 10 9 9 17/30 Write block 210 len 20480 data: 14 13 13 10 11 18/30 Read block 70 len 8192 19/30 Read block 396 len 36864 20/30 Read block 226 len 16384 21/30 Write block 345 len 4096 data: 12 22/30 Read block 544 len 40960 23/30 Read block 184 len 16384 24/30 Read block 135 len 20480 25/30 Flush 26/30 Write block 432 len 36864 data: 13 11 16 17 16 16 17 18 17 27/30 Read block 127 len 8192 28/30 Write block 529 len 40960 data: 10 12 11 12 12 15 14 15 16 15 29/30 Write block 394 len 20480 data: 14 18 18 18 17 30/30 Write block 391 len 20480 data: 14 14 17 15 19 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 30 3 36 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:18.696557259Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1916} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 302 302 302 F F F 1 020-039 175 175 175 302 302 302 F F F 2 040-059 174 174 174 301 301 301 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 176 169 176 303 289 303 F F F <--- 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 171 176 303 293 303 F F F <--- 8 160-179 176 173 176 303 298 303 F F F <--- 9 180-199 173 173 173 298 298 298 F F F 10 200-219 176 174 176 303 300 303 F F F <--- 11 220-239 170 170 170 290 290 290 F F F 12 240-259 175 175 175 302 302 302 F F F 13 260-279 176 163 176 303 281 303 F F F <--- 14 280-299 169 169 169 289 289 289 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 176 168 176 303 288 303 F F F <--- 18 360-379 163 163 163 281 281 281 F F F 19 380-399 174 174 174 301 301 301 T F T <--- 20 400-419 171 171 171 293 293 293 F F F 21 420-439 174 174 174 299 299 299 T F T <--- 22 440-459 176 171 176 303 293 303 T F T <--- 23 460-479 176 173 176 303 298 303 F F F <--- 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 173 173 173 298 298 298 T F T <--- 27 540-559 174 174 174 299 299 299 F F F 28 560-579 174 174 174 299 299 299 F F F 29 580-599 176 174 176 303 300 303 F F F <--- Max gen: 176, Max flush: 303 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 302 302 302 F F F 1 020-039 175 175 175 302 302 302 F F F 2 040-059 174 174 174 301 301 301 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 176 176 176 303 303 303 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 176 176 176 303 303 303 F F F 11 220-239 170 170 170 290 290 290 F F F 12 240-259 175 175 175 302 302 302 F F F 13 260-279 176 176 176 303 303 303 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 177 177 177 304 304 304 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 177 177 177 304 304 304 F F F 22 440-459 177 177 177 304 304 304 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 177 177 177 304 304 304 F F F 27 540-559 174 174 174 299 299 299 F F F 28 560-579 174 174 174 299 299 299 F F F 29 580-599 176 176 176 303 303 303 F F F Max gen: 177, Max flush: 304 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.108497615Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:23.151864313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.152279997Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.183206859Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.183874833Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.183935931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.18395617Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.183991679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} {"msg":"Crucible 1fe7b0e1-f545-481e-9d43-39e2e56af452 has session id: 0825f98b-55dc-46f8-9074-4dcebbb5dbd7","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.184521759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Upstairs opts: Upstairs UUID: 1fe7b0e1-f545-481e-9d43-39e2e56af452, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.184595656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Crucible stats registered with UUID: 1fe7b0e1-f545-481e-9d43-39e2e56af452","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.184630004Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:23.184650584Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.184768309Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.184815177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.184841727Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185021729Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185043369Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} {"msg":"1fe7b0e1-f545-481e-9d43-39e2e56af452 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185121286Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185142275Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"0","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185161304Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"1","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185180034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"2","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185197253Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185214672Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185231721Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63638)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185330778Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49125)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185377816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35033)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.185484912Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.186686816Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"1","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.186720365Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"2","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.18683978Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"0","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187390719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187417088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[0]R flush_numbers[0..12]: [302, 302, 301, 301, 303, 286, 299, 303, 303, 298, 303, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187449547Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[0]R generation[0..12]: [175, 175, 174, 174, 176, 166, 174, 176, 176, 173, 176, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187469186Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187488806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[1]R flush_numbers[0..12]: [302, 302, 301, 301, 303, 286, 299, 303, 303, 298, 303, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187507645Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[1]R generation[0..12]: [175, 175, 174, 174, 176, 166, 174, 176, 176, 173, 176, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187526394Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187545853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[2]R flush_numbers[0..12]: [302, 302, 301, 301, 303, 286, 299, 303, 303, 298, 303, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187564853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[2]R generation[0..12]: [175, 175, 174, 174, 176, 166, 174, 176, 176, 173, 176, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187583542Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187602891Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187619171Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.18763613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187652269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187671379Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"":"downstairs","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:05:23.187688698Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187709657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"0","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187726797Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"1","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187743416Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"client":"2","":"downstairs","session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187760425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187776315Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"1fe7b0e1-f545-481e-9d43-39e2e56af452 is now active with session: 0825f98b-55dc-46f8-9074-4dcebbb5dbd7","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187794274Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187809443Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924,"session_id":"0825f98b-55dc-46f8-9074-4dcebbb5dbd7"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187825043Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} {"msg":"Activated sub_volume 1fe7b0e1-f545-481e-9d43-39e2e56af452","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:23.187838862Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 16384 data: 20 18 17 19 02/30 Write block 433 len 24576 data: 12 17 18 17 17 18 03/30 Read block 333 len 28672 04/30 Write block 8 len 36864 data: 16 15 12 12 14 11 12 12 13 05/30 Read block 263 len 36864 06/30 Read block 394 len 24576 07/30 Read block 486 len 28672 08/30 Read block 384 len 36864 09/30 Read block 28 len 4096 10/30 Read block 183 len 8192 11/30 Write block 26 len 4096 data: 16 12/30 Read block 573 len 4096 13/30 Read block 390 len 4096 14/30 Write block 82 len 8192 data: 11 15 15/30 Read block 322 len 20480 16/30 Read block 173 len 8192 17/30 Read block 307 len 24576 18/30 Read block 439 len 32768 19/30 Write block 207 len 32768 data: 19 17 16 15 14 14 11 12 20/30 Write block 551 len 12288 data: 14 12 15 21/30 Write block 567 len 36864 data: 9 11 13 16 19 18 21 23 25 22/30 Read block 489 len 12288 23/30 Read block 222 len 28672 24/30 Write block 551 len 24576 data: 15 13 16 16 14 16 25/30 Read block 566 len 40960 26/30 Read block 414 len 20480 27/30 Write block 258 len 40960 data: 13 14 12 15 13 10 10 10 10 10 28/30 Read block 210 len 16384 29/30 Read block 204 len 24576 30/30 Read block 232 len 20480 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 4 30 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:23.207183395Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1924} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 302 302 302 T T T 1 020-039 175 175 175 302 302 302 F T T <--- 2 040-059 174 174 174 301 301 301 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 176 176 176 303 303 303 F T T <--- 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 176 176 176 303 303 303 T T T 11 220-239 170 170 170 290 290 290 F F F 12 240-259 175 175 175 302 302 302 F T T <--- 13 260-279 176 176 176 303 303 303 F T T <--- 14 280-299 169 169 169 289 289 289 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 177 177 177 304 304 304 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 177 177 177 304 304 304 T T T 22 440-459 177 177 177 304 304 304 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 177 177 177 304 304 304 F F F 27 540-559 174 174 174 299 299 299 F T T <--- 28 560-579 174 174 174 299 299 299 F T T <--- 29 580-599 176 176 176 303 303 303 F F F Max gen: 177, Max flush: 304 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 89 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 210c210,211 < 18, --- > 19, > 17, 214,216c215 < 13, < 13, < 10, --- > 14, 217a217 > 12, diff found after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 305 305 305 F F F 1 020-039 178 178 178 305 305 305 F F F 2 040-059 174 174 174 301 301 301 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 178 178 178 305 305 305 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 178 178 178 305 305 305 F F F 11 220-239 170 170 170 290 290 290 F F F 12 240-259 178 178 178 305 305 305 F F F 13 260-279 178 178 178 305 305 305 F F F 14 280-299 169 169 169 289 289 289 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 177 177 177 304 304 304 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 178 178 178 305 305 305 F F F 22 440-459 177 177 177 304 304 304 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F F F 26 520-539 177 177 177 304 304 304 F F F 27 540-559 178 178 178 305 305 305 F F F 28 560-579 178 178 178 305 305 305 F F F 29 580-599 176 176 176 303 303 303 F F F Max gen: 178, Max flush: 305 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.638504113Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:27.681034053Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.68138577Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.711928676Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.712567492Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.71261591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.712636439Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.712658989Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} {"msg":"Crucible 58c35c79-e70e-41c5-95fb-6125ac3d8080 has session id: c7d1b85b-61e2-4cff-a604-cf81437f4cc1","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713202798Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Upstairs opts: Upstairs UUID: 58c35c79-e70e-41c5-95fb-6125ac3d8080, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713256256Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Crucible stats registered with UUID: 58c35c79-e70e-41c5-95fb-6125ac3d8080","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713286035Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:27.713302854Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713505556Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713546585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"2","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713566214Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"1","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713727428Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713756887Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} {"msg":"58c35c79-e70e-41c5-95fb-6125ac3d8080 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713798345Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713818204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"0","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713835944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"1","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713856543Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"2","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713873662Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"2","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713891031Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.713908101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"1","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35392)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.714174361Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"1","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43844)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.714214869Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"2","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35123)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.714241878Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.715429723Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"1","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.715484251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"0","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.715524859Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"2","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.715978142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716107777Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[0]R flush_numbers[0..12]: [305, 305, 301, 301, 305, 286, 299, 303, 303, 298, 305, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716147436Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[0]R generation[0..12]: [178, 178, 174, 174, 178, 166, 174, 176, 176, 173, 178, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716168475Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716193104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[1]R flush_numbers[0..12]: [305, 305, 301, 301, 305, 286, 299, 303, 303, 298, 305, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716212743Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[1]R generation[0..12]: [178, 178, 174, 174, 178, 166, 174, 176, 176, 173, 178, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716231642Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716251412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[2]R flush_numbers[0..12]: [305, 305, 301, 301, 305, 286, 299, 303, 303, 298, 305, 290]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716270161Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[2]R generation[0..12]: [178, 178, 174, 174, 178, 166, 174, 176, 176, 173, 178, 170]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.71628877Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716308279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Max found gen is 179","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716324489Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Generation requested: 180 >= found:179","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716341468Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716363427Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716379377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"":"downstairs","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716396576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716411786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"0","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716428855Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"1","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716446084Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"client":"2","":"downstairs","session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716465113Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716481283Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"58c35c79-e70e-41c5-95fb-6125ac3d8080 is now active with session: c7d1b85b-61e2-4cff-a604-cf81437f4cc1","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716499052Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716514412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935,"session_id":"c7d1b85b-61e2-4cff-a604-cf81437f4cc1"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.716530561Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} {"msg":"Activated sub_volume 58c35c79-e70e-41c5-95fb-6125ac3d8080","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:27.71654439Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 49 len 36864 data: 18 16 16 17 17 17 12 11 11 02/30 Read block 518 len 20480 03/30 Write block 283 len 36864 data: 11 9 11 11 13 12 13 13 14 04/30 Write block 542 len 40960 data: 12 12 14 11 12 13 15 13 14 16 05/30 Write block 448 len 28672 data: 14 13 11 12 10 10 9 06/30 Read block 304 len 16384 07/30 Write block 239 len 20480 data: 11 8 8 10 10 08/30 Read block 386 len 28672 09/30 Read block 295 len 32768 10/30 Read block 429 len 36864 11/30 Write block 201 len 24576 data: 21 19 18 20 20 17 12/30 Read block 83 len 4096 13/30 Flush 14/30 Write block 435 len 8192 data: 19 18 15/30 Write block 264 len 36864 data: 11 11 11 11 10 13 12 14 14 16/30 Read block 585 len 40960 17/30 Write block 549 len 36864 data: 14 15 17 14 17 17 15 17 19 18/30 Read block 105 len 36864 19/30 Read block 19 len 40960 20/30 Write block 536 len 4096 data: 16 21/30 Read block 276 len 28672 22/30 Read block 129 len 36864 23/30 Write block 507 len 24576 data: 9 9 6 5 5 5 24/30 Write block 11 len 16384 data: 13 15 12 13 25/30 Write block 238 len 8192 data: 11 12 26/30 Read block 240 len 8192 27/30 Read block 300 len 4096 28/30 Write block 236 len 28672 data: 14 14 12 13 9 9 11 29/30 Read block 22 len 32768 30/30 Read block 183 len 40960 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 3 29 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:27.736842157Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1935} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 305 305 305 F T T <--- 1 020-039 178 178 178 305 305 305 F F F 2 040-059 174 180 180 301 306 306 T F F <--- 3 060-079 174 174 174 301 301 301 F F F 4 080-099 178 178 178 305 305 305 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 178 180 180 305 306 306 F F F <--- 11 220-239 170 180 180 290 306 306 F T T <--- 12 240-259 178 180 180 305 306 306 F T T <--- 13 260-279 178 178 178 305 305 305 F T T <--- 14 280-299 169 180 180 289 306 306 T F F <--- 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 177 177 177 304 304 304 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 178 178 178 305 305 305 F T T <--- 22 440-459 177 180 180 304 306 306 F F F <--- 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 173 173 173 298 298 298 F T T <--- 26 520-539 177 177 177 304 304 304 F T T <--- 27 540-559 178 180 180 305 306 306 F T T <--- 28 560-579 178 178 178 305 305 305 F F F 29 580-599 176 176 176 303 303 303 F F F Max gen: 180, Max flush: 306 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 307 307 307 F F F 1 020-039 178 178 178 305 305 305 F F F 2 040-059 180 180 180 306 306 306 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 178 178 178 305 305 305 F F F 5 100-119 166 166 166 286 286 286 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 180 180 180 306 306 306 F F F 11 220-239 181 181 181 307 307 307 F F F 12 240-259 181 181 181 307 307 307 F F F 13 260-279 181 181 181 307 307 307 F F F 14 280-299 180 180 180 306 306 306 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 170 170 292 292 292 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 177 177 177 304 304 304 F F F 20 400-419 171 171 171 293 293 293 F F F 21 420-439 181 181 181 307 307 307 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 181 181 181 307 307 307 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 178 178 178 305 305 305 F F F 29 580-599 176 176 176 303 303 303 F F F Max gen: 181, Max flush: 307 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.165725588Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:32.209956243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.210346198Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.241294689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.241963103Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242012412Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242028251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.24205036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} {"msg":"Crucible 4a8feba5-a92f-45e0-a408-f7f7300a8b66 has session id: 76be7089-3005-474c-bcca-84cccd576661","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242525152Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Upstairs opts: Upstairs UUID: 4a8feba5-a92f-45e0-a408-f7f7300a8b66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242671836Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Crucible stats registered with UUID: 4a8feba5-a92f-45e0-a408-f7f7300a8b66","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242700826Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:32.242717595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242920047Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242962716Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"0","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.242981825Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"1","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243132149Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243155318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} {"msg":"4a8feba5-a92f-45e0-a408-f7f7300a8b66 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243221096Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243241605Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"0","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243258974Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"1","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243275664Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"2","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243292313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243309242Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"0","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243336021Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"1","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"ds_connection connected from Ok(127.0.0.1:43834)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243434278Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"1","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"ds_connection connected from Ok(127.0.0.1:33689)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.243515864Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"ds_connection connected from Ok(127.0.0.1:41487)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.24363093Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"io task","client":"0","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.244775296Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"1","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.244809135Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"2","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.244845714Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"0","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245352354Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245434981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[0]R flush_numbers[0..12]: [307, 305, 306, 301, 305, 286, 299, 303, 303, 298, 306, 307]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245541977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[0]R generation[0..12]: [181, 178, 180, 174, 178, 166, 174, 176, 176, 173, 180, 181]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245566566Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245586485Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[1]R flush_numbers[0..12]: [307, 305, 306, 301, 305, 286, 299, 303, 303, 298, 306, 307]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245605515Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[1]R generation[0..12]: [181, 178, 180, 174, 178, 166, 174, 176, 176, 173, 180, 181]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245624264Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245644023Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[2]R flush_numbers[0..12]: [307, 305, 306, 301, 305, 286, 299, 303, 303, 298, 306, 307]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245667493Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[2]R generation[0..12]: [181, 178, 180, 174, 178, 166, 174, 176, 176, 173, 180, 181]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245686792Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245706231Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.24572254Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.24573946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Next flush: 308","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2026-02-15T00:05:32.245764409Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245780768Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"":"downstairs","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245797568Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245812817Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"0","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245829576Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"1","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245846226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"client":"2","":"downstairs","session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245863105Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245879204Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"4a8feba5-a92f-45e0-a408-f7f7300a8b66 is now active with session: 76be7089-3005-474c-bcca-84cccd576661","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245897074Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245912173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943,"session_id":"76be7089-3005-474c-bcca-84cccd576661"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245928303Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} {"msg":"Activated sub_volume 4a8feba5-a92f-45e0-a408-f7f7300a8b66","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:32.245944002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 580 len 36864 data: 15 13 12 14 14 15 16 14 11 03/30 Write block 382 len 20480 data: 10 9 9 11 10 04/30 Flush 05/30 Write block 507 len 4096 data: 10 06/30 Write block 332 len 32768 data: 10 11 10 10 13 15 15 14 07/30 Write block 421 len 16384 data: 14 16 15 20 08/30 Read block 214 len 40960 09/30 Read block 545 len 32768 10/30 Write block 111 len 28672 data: 8 10 12 12 13 15 18 11/30 Write block 23 len 8192 data: 15 18 12/30 Flush 13/30 Flush 14/30 Write block 80 len 16384 data: 11 9 12 16 15/30 Write block 505 len 40960 data: 16 13 11 10 7 6 6 6 8 9 16/30 Read block 174 len 20480 17/30 Write block 427 len 4096 data: 22 18/30 Read block 554 len 36864 19/30 Write block 89 len 12288 data: 15 11 10 20/30 Read block 488 len 24576 21/30 Read block 510 len 28672 22/30 Read block 394 len 40960 23/30 Read block 105 len 32768 24/30 Read block 172 len 16384 25/30 Read block 7 len 16384 26/30 Write block 329 len 24576 data: 11 11 11 11 12 11 27/30 Write block 573 len 36864 data: 22 24 26 24 21 16 15 16 14 28/30 Read block 482 len 40960 29/30 Write block 416 len 32768 data: 10 12 12 12 13 15 17 16 30/30 Read block 419 len 20480 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 1 29 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:32.265715069Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1943} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 307 307 307 F F F 1 020-039 178 182 182 305 310 310 F F F <--- 2 040-059 180 180 180 306 306 306 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 178 178 178 305 305 305 F T T <--- 5 100-119 166 182 182 286 310 310 F F F <--- 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 180 180 180 306 306 306 F F F 11 220-239 181 181 181 307 307 307 F F F 12 240-259 181 181 181 307 307 307 F F F 13 260-279 181 181 181 307 307 307 F F F 14 280-299 180 180 180 306 306 306 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 170 182 182 292 310 310 F T T <--- 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 177 182 182 304 309 309 F F F <--- 20 400-419 171 171 171 293 293 293 F T T <--- 21 420-439 181 182 182 307 310 310 F T T <--- 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 181 182 182 307 310 310 F T T <--- 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 178 178 178 305 305 305 F T T <--- 29 580-599 176 182 182 303 309 309 T T T <--- Max gen: 182, Max flush: 310 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 307 307 307 F F F 1 020-039 182 182 182 310 310 310 F F F 2 040-059 180 180 180 306 306 306 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 183 183 183 311 311 311 F F F 5 100-119 182 182 182 310 310 310 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 176 176 176 303 303 303 F F F 8 160-179 176 176 176 303 303 303 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 180 180 180 306 306 306 F F F 11 220-239 181 181 181 307 307 307 F F F 12 240-259 181 181 181 307 307 307 F F F 13 260-279 181 181 181 307 307 307 F F F 14 280-299 180 180 180 306 306 306 F F F 15 300-319 175 175 175 302 302 302 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 163 163 163 281 281 281 F F F 19 380-399 182 182 182 309 309 309 F F F 20 400-419 183 183 183 311 311 311 F F F 21 420-439 183 183 183 311 311 311 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 183 183 183 311 311 311 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 183, Max flush: 311 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.689248453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.73370759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.734453742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.73450318Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.734519679Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.734542218Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} {"msg":"Crucible aa0f73f1-e773-4b52-b59a-4f4901c14a87 has session id: f1a47fa1-6a8d-474e-81e9-9a81633f918b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735073148Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Upstairs opts: Upstairs UUID: aa0f73f1-e773-4b52-b59a-4f4901c14a87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735142065Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Crucible stats registered with UUID: aa0f73f1-e773-4b52-b59a-4f4901c14a87","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735168974Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:36.735185534Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735331718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"0","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735372236Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735391306Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"2","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735684725Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735705624Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} {"msg":"aa0f73f1-e773-4b52-b59a-4f4901c14a87 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735789461Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.73581655Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"0","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735842059Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"1","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735873177Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"2","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735904746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"0","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735933675Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"2","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.735961034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50862)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.736018142Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"2","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36083)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.73605651Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61840)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.736090459Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"io task","client":"0","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.737307453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"0","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.737451577Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"2","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.737860602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738067334Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738117432Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"1","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[0]R flush_numbers[0..12]: [307, 310, 306, 301, 311, 310, 299, 303, 303, 298, 306, 307]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738628652Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[0]R generation[0..12]: [181, 182, 180, 174, 183, 182, 174, 176, 176, 173, 180, 181]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738653671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738673801Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[1]R flush_numbers[0..12]: [307, 310, 306, 301, 311, 310, 299, 303, 303, 298, 306, 307]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.73869309Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[1]R generation[0..12]: [181, 182, 180, 174, 183, 182, 174, 176, 176, 173, 180, 181]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738712219Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738731959Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[2]R flush_numbers[0..12]: [307, 310, 306, 301, 311, 310, 299, 303, 303, 298, 306, 307]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738751718Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[2]R generation[0..12]: [181, 182, 180, 174, 183, 182, 174, 176, 176, 173, 180, 181]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738770407Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738790186Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738813456Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738888513Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738906762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738924141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"":"downstairs","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738944131Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.7389598Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"0","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738976759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"1","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.738993618Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"client":"2","":"downstairs","session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.739010558Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.739026657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"aa0f73f1-e773-4b52-b59a-4f4901c14a87 is now active with session: f1a47fa1-6a8d-474e-81e9-9a81633f918b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.739044477Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2026-02-15T00:05:36.739075086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951,"session_id":"f1a47fa1-6a8d-474e-81e9-9a81633f918b"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.739091625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} {"msg":"Activated sub_volume aa0f73f1-e773-4b52-b59a-4f4901c14a87","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:36.739105604Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 28672 data: 12 11 11 16 11 12 10 02/30 Read block 418 len 8192 03/30 Write block 296 len 12288 data: 14 14 18 04/30 Write block 362 len 32768 data: 9 10 9 13 12 12 17 12 05/30 Write block 32 len 20480 data: 11 10 8 8 6 06/30 Read block 426 len 36864 07/30 Flush 08/30 Write block 104 len 20480 data: 15 13 12 12 10 09/30 Write block 143 len 8192 data: 13 12 10/30 Read block 243 len 32768 11/30 Write block 217 len 24576 data: 14 12 8 9 9 13 12/30 Read block 107 len 4096 13/30 Write block 170 len 4096 data: 13 14/30 Read block 230 len 8192 15/30 Read block 192 len 32768 16/30 Write block 297 len 16384 data: 15 19 18 17 17/30 Write block 301 len 32768 data: 19 21 19 16 18 17 17 15 18/30 Write block 229 len 12288 data: 10 13 14 19/30 Write block 397 len 40960 data: 19 18 15 13 11 12 16 15 16 17 20/30 Read block 453 len 28672 21/30 Write block 262 len 4096 data: 14 22/30 Flush 23/30 Flush 24/30 Write block 293 len 20480 data: 15 14 15 15 16 25/30 Write block 309 len 12288 data: 18 15 18 26/30 Read block 357 len 4096 27/30 Read block 281 len 24576 28/30 Write block 9 len 4096 data: 16 29/30 Read block 226 len 36864 30/30 Write block 111 len 36864 data: 9 11 13 13 14 16 19 18 15 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 20 1 22 Done 22 3 22 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1006)) Some(JobId(1022)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:36.757292692Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1951} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 307 307 307 T F T <--- 1 020-039 184 184 184 312 312 312 F F F 2 040-059 180 180 180 306 306 306 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 183 183 183 311 311 311 F F F 5 100-119 184 182 184 313 310 313 T T T <--- 6 120-139 174 174 174 299 299 299 F F F 7 140-159 184 176 184 313 303 313 F T F <--- 8 160-179 184 176 184 313 303 313 F F F <--- 9 180-199 173 173 173 298 298 298 F F F 10 200-219 184 180 184 313 306 313 F F F <--- 11 220-239 184 181 184 313 307 313 F F F <--- 12 240-259 181 181 181 307 307 307 F F F 13 260-279 184 181 184 313 307 313 F F F <--- 14 280-299 184 184 184 313 312 313 T F T <--- 15 300-319 184 175 184 313 302 313 T F T <--- 16 320-339 183 183 183 311 311 311 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 184 184 184 312 312 312 F F F 19 380-399 184 182 184 313 309 313 F F F <--- 20 400-419 184 183 184 313 311 313 F F F <--- 21 420-439 183 183 183 311 311 311 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 183 183 183 311 311 311 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 184, Max flush: 313 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 184 184 184 312 312 312 F F F 2 040-059 180 180 180 306 306 306 F F F 3 060-079 174 174 174 301 301 301 F F F 4 080-099 183 183 183 311 311 311 F F F 5 100-119 185 185 185 314 314 314 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 184 184 184 313 313 313 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 184 184 184 313 313 313 F F F 11 220-239 184 184 184 313 313 313 F F F 12 240-259 181 181 181 307 307 307 F F F 13 260-279 184 184 184 313 313 313 F F F 14 280-299 185 185 185 314 314 314 F F F 15 300-319 185 185 185 314 314 314 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 184 184 184 312 312 312 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 184 184 184 313 313 313 F F F 21 420-439 183 183 183 311 311 311 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 181 181 181 307 307 307 F F F 28 560-579 183 183 183 311 311 311 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 185, Max flush: 314 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.171751101Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.215857931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.216576393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.216636741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.21665763Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.216692309Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} {"msg":"Crucible 16a766e7-a125-4fd8-9850-63e35bf17925 has session id: 6b6321e8-12ad-4ce5-819b-a40566ba1a86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217208609Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Upstairs opts: Upstairs UUID: 16a766e7-a125-4fd8-9850-63e35bf17925, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217258668Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Crucible stats registered with UUID: 16a766e7-a125-4fd8-9850-63e35bf17925","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217279317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:41.217295506Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217523517Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217571316Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217602454Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"1","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217836266Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217865595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} {"msg":"16a766e7-a125-4fd8-9850-63e35bf17925 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217929162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217950191Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"0","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.217967551Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"1","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.21799621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218028198Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218062297Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218095226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"1","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:41.218169203Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"ds_connection connected from Ok(127.0.0.1:59347)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218216631Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:41.218343476Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"ds_connection connected from Ok(127.0.0.1:63200)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218380915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"1","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:41.218405504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218428313Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.218462302Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.219251862Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"1","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.219336719Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"0","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.219842609Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:41.219924146Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.219491983Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"ds_connection connected from Ok(127.0.0.1:59491)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.219691325Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.221494856Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[0]R flush_numbers[0..12]: [314, 312, 306, 301, 311, 314, 299, 313, 313, 298, 313, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222166741Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[0]R generation[0..12]: [185, 184, 180, 174, 183, 185, 174, 184, 184, 173, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222204939Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222231278Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[1]R flush_numbers[0..12]: [314, 312, 306, 301, 311, 314, 299, 313, 313, 298, 313, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222255418Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[1]R generation[0..12]: [185, 184, 180, 174, 183, 185, 174, 184, 184, 173, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222278977Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222303396Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[2]R flush_numbers[0..12]: [314, 312, 306, 301, 311, 314, 299, 313, 313, 298, 313, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222326915Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[2]R generation[0..12]: [185, 184, 180, 174, 183, 185, 174, 184, 184, 173, 184, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222350154Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:05:51.222374523Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222397252Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222414241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222430141Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.22244616Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"":"downstairs","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222463099Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222478389Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"0","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222497348Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"1","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222514317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"client":"2","":"downstairs","session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222531267Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222547246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"16a766e7-a125-4fd8-9850-63e35bf17925 is now active with session: 6b6321e8-12ad-4ce5-819b-a40566ba1a86","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222565006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222580225Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959,"session_id":"6b6321e8-12ad-4ce5-819b-a40566ba1a86"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222596265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} {"msg":"Activated sub_volume 16a766e7-a125-4fd8-9850-63e35bf17925","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:05:51.222610524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 16384 data: 12 11 8 9 02/30 Write block 202 len 28672 data: 20 19 21 21 18 19 17 03/30 Read block 224 len 28672 04/30 Read block 311 len 28672 05/30 Read block 131 len 28672 06/30 Read block 293 len 8192 07/30 Read block 437 len 12288 08/30 Read block 573 len 16384 09/30 Read block 472 len 40960 10/30 Write block 575 len 4096 data: 27 11/30 Write block 548 len 28672 data: 16 15 16 18 15 18 18 12/30 Read block 178 len 36864 13/30 Write block 164 len 36864 data: 11 11 8 9 11 9 14 13 15 14/30 Write block 590 len 4096 data: 10 15/30 Write block 78 len 40960 data: 14 12 12 10 13 17 16 19 19 16 16/30 Write block 375 len 8192 data: 12 11 17/30 Write block 211 len 16384 data: 14 14 11 12 18/30 Read block 358 len 28672 19/30 Flush 20/30 Write block 213 len 8192 data: 12 13 21/30 Write block 399 len 28672 data: 16 14 12 13 17 16 17 22/30 Read block 467 len 36864 23/30 Read block 451 len 28672 24/30 Read block 171 len 8192 25/30 Read block 374 len 32768 26/30 Write block 25 len 4096 data: 17 27/30 Read block 340 len 12288 28/30 Write block 424 len 40960 data: 21 18 18 23 21 17 15 18 14 13 29/30 Write block 35 len 24576 data: 9 7 8 11 11 13 30/30 Read block 203 len 16384 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 24 25 Done 29 30 6 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:05:51.241592611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1959} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 184 184 184 312 312 312 T T F <--- 2 040-059 180 180 180 306 306 306 T T F <--- 3 060-079 186 186 174 315 315 301 F F F <--- 4 080-099 186 186 183 315 315 311 F F F <--- 5 100-119 185 185 185 314 314 314 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 186 186 184 315 315 313 F F F <--- 9 180-199 173 173 173 298 298 298 F F F 10 200-219 186 186 184 315 315 313 T T T <--- 11 220-239 184 184 184 313 313 313 F F F 12 240-259 186 186 181 315 315 307 F F T <--- 13 260-279 184 184 184 313 313 313 F F F 14 280-299 185 185 185 314 314 314 F F F 15 300-319 185 185 185 314 314 314 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 186 186 184 315 315 312 F F F <--- 19 380-399 184 184 184 313 313 313 T T F <--- 20 400-419 184 184 184 313 313 313 T T F <--- 21 420-439 183 183 183 311 311 311 T T F <--- 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 186 186 181 315 315 307 F F F <--- 28 560-579 186 186 183 315 315 311 F F F <--- 29 580-599 186 186 183 315 315 311 F F F <--- Max gen: 186, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 187 187 187 316 316 316 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 186 186 186 315 315 315 F F F 4 080-099 186 186 186 315 315 315 F F F 5 100-119 185 185 185 314 314 314 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 186 186 186 315 315 315 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 187 187 187 316 316 316 F F F 11 220-239 184 184 184 313 313 313 F F F 12 240-259 186 186 186 315 315 315 F F F 13 260-279 184 184 184 313 313 313 F F F 14 280-299 185 185 185 314 314 314 F F F 15 300-319 185 185 185 314 314 314 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 176 176 176 303 303 303 F F F 18 360-379 186 186 186 315 315 315 F F F 19 380-399 187 187 187 316 316 316 F F F 20 400-419 187 187 187 316 316 316 F F F 21 420-439 187 187 187 316 316 316 F F F 22 440-459 180 180 180 306 306 306 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 186 186 186 315 315 315 F F F 28 560-579 186 186 186 315 315 315 F F F 29 580-599 186 186 186 315 315 315 F F F Max gen: 187, Max flush: 316 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.652113431Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.698251744Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.698965196Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699013415Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699029944Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699051873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} {"msg":"Crucible 75da43e2-f182-4f1e-abb0-f7afdafbfb41 has session id: 6dd60206-16dd-4fd1-be15-4f42683ee62f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.69964731Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Upstairs opts: Upstairs UUID: 75da43e2-f182-4f1e-abb0-f7afdafbfb41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699700828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Crucible stats registered with UUID: 75da43e2-f182-4f1e-abb0-f7afdafbfb41","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699721888Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:05.699738337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699943809Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.699982827Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"0","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700003457Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700142972Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700164431Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} {"msg":"75da43e2-f182-4f1e-abb0-f7afdafbfb41 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700232808Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700252987Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"0","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700270037Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700297206Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"2","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700326345Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700358193Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"0","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700392192Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59426)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.70044329Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44519)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700475699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"0","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:05.700574805Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:05.700664812Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:05.70071385Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700812476Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.700855384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.701674173Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"2","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.701791179Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"0","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.702172544Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:05.702313169Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.701401393Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33096)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.701539358Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.703238604Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[0]R flush_numbers[0..12]: [314, 316, 316, 315, 315, 314, 299, 313, 315, 298, 316, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.703920007Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[0]R generation[0..12]: [185, 187, 187, 186, 186, 185, 174, 184, 186, 173, 187, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.703957846Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.703983895Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[1]R flush_numbers[0..12]: [314, 316, 316, 315, 315, 314, 299, 313, 315, 298, 316, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704007724Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[1]R generation[0..12]: [185, 187, 187, 186, 186, 185, 174, 184, 186, 173, 187, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704031483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704055972Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[2]R flush_numbers[0..12]: [314, 316, 316, 315, 315, 314, 299, 313, 315, 298, 316, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704079981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[2]R generation[0..12]: [185, 187, 187, 186, 186, 185, 174, 184, 186, 173, 187, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704103391Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704127779Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704147949Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704173208Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704193537Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704213786Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"":"downstairs","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704235046Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704253935Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"0","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704277174Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"1","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704300013Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"client":"2","":"downstairs","session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704321622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704341841Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"75da43e2-f182-4f1e-abb0-f7afdafbfb41 is now active with session: 6dd60206-16dd-4fd1-be15-4f42683ee62f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704364081Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.70438748Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967,"session_id":"6dd60206-16dd-4fd1-be15-4f42683ee62f"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704408209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} {"msg":"Activated sub_volume 75da43e2-f182-4f1e-abb0-f7afdafbfb41","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:15.704425248Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 4096 data: 14 02/30 Read block 478 len 12288 03/30 Write block 444 len 16384 data: 12 16 16 15 04/30 Flush 05/30 Write block 327 len 12288 data: 9 11 12 06/30 Flush 07/30 Write block 97 len 36864 data: 11 11 11 16 17 17 17 16 14 08/30 Read block 145 len 20480 09/30 Write block 573 len 16384 data: 23 25 28 25 10/30 Read block 476 len 24576 11/30 Read block 219 len 24576 12/30 Write block 358 len 28672 data: 7 7 7 9 10 11 10 13/30 Write block 415 len 12288 data: 10 11 13 14/30 Write block 54 len 8192 data: 18 13 15/30 Write block 207 len 20480 data: 20 18 16 15 15 16/30 Read block 301 len 24576 17/30 Write block 420 len 12288 data: 14 16 18 18/30 Write block 52 len 24576 data: 18 18 19 14 12 12 19/30 Read block 212 len 16384 20/30 Read block 593 len 4096 21/30 Read block 421 len 4096 22/30 Write block 558 len 36864 data: 19 23 19 17 17 18 15 10 8 23/30 Read block 323 len 16384 24/30 Write block 586 len 32768 data: 17 15 12 10 11 9 10 11 25/30 Write block 272 len 4096 data: 15 26/30 Write block 160 len 12288 data: 14 15 16 27/30 Write block 553 len 36864 data: 19 19 16 18 20 20 24 20 18 28/30 Write block 203 len 40960 data: 20 22 22 19 21 19 17 16 16 15 29/30 Read block 240 len 8192 30/30 Read block 567 len 28672 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Read 3 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) None Some(JobId(1005)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:15.722427573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1967} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 187 187 187 316 316 316 F F F 2 040-059 187 187 187 316 316 316 T F T <--- 3 060-079 186 186 186 315 315 315 F F F 4 080-099 186 186 186 315 315 315 T F T <--- 5 100-119 185 185 185 314 314 314 T F T <--- 6 120-139 174 174 174 299 299 299 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 186 186 186 315 315 315 T F T <--- 9 180-199 173 173 173 298 298 298 F F F 10 200-219 187 187 187 316 316 316 T F T <--- 11 220-239 184 184 184 313 313 313 F F F 12 240-259 186 186 186 315 315 315 F F F 13 260-279 184 184 184 313 313 313 T F T <--- 14 280-299 185 185 185 314 314 314 F F F 15 300-319 185 185 185 314 314 314 F F F 16 320-339 188 183 188 318 311 318 F F F <--- 17 340-359 176 176 176 303 303 303 T F T <--- 18 360-379 186 186 186 315 315 315 T F T <--- 19 380-399 187 187 187 316 316 316 F F F 20 400-419 187 187 187 316 316 316 T F T <--- 21 420-439 187 187 187 316 316 316 T F T <--- 22 440-459 188 180 188 317 306 317 F F F <--- 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 186 186 186 315 315 315 T F T <--- 28 560-579 186 186 186 315 315 315 T F T <--- 29 580-599 186 186 186 315 315 315 T F T <--- Max gen: 188, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 187 187 187 316 316 316 F F F 2 040-059 189 189 189 319 319 319 F F F 3 060-079 186 186 186 315 315 315 F F F 4 080-099 189 189 189 319 319 319 F F F 5 100-119 189 189 189 319 319 319 F F F 6 120-139 174 174 174 299 299 299 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 189 189 189 319 319 319 F F F 9 180-199 173 173 173 298 298 298 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 184 184 184 313 313 313 F F F 12 240-259 186 186 186 315 315 315 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 185 185 185 314 314 314 F F F 15 300-319 185 185 185 314 314 314 F F F 16 320-339 188 188 188 318 318 318 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 189 189 189 319 319 319 F F F 19 380-399 187 187 187 316 316 316 F F F 20 400-419 189 189 189 319 319 319 F F F 21 420-439 189 189 189 319 319 319 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 189 189 189 319 319 319 F F F 28 560-579 189 189 189 319 319 319 F F F 29 580-599 189 189 189 319 319 319 F F F Max gen: 189, Max flush: 319 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.14901656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:20.19311257Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.193507975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.225312773Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226002597Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226050945Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226067975Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226094334Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} {"msg":"Crucible e59e40eb-8e23-4535-a935-cb4d2385650f has session id: 1539e30a-d4e4-43ae-b072-cea0bb242f7d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226688511Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Upstairs opts: Upstairs UUID: e59e40eb-8e23-4535-a935-cb4d2385650f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.22672668Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Crucible stats registered with UUID: e59e40eb-8e23-4535-a935-cb4d2385650f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226745989Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:20.226762068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.226998029Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"0","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227039368Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"1","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227059547Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227268689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227293168Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} {"msg":"e59e40eb-8e23-4535-a935-cb4d2385650f active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227358966Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227379235Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"0","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227396434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"1","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227413573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"2","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227430133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227451272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"0","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227468471Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"1","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59709)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227573677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"0","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36457)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227618226Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63870)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.227652134Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"io task","client":"1","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.228870088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"1","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.228974504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"0","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229049611Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"2","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229468215Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229578961Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[0]R flush_numbers[0..12]: [314, 316, 319, 315, 319, 319, 299, 313, 319, 298, 319, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229654838Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[0]R generation[0..12]: [185, 187, 189, 186, 189, 189, 174, 184, 189, 173, 189, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229679677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229699656Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[1]R flush_numbers[0..12]: [314, 316, 319, 315, 319, 319, 299, 313, 319, 298, 319, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229718886Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[1]R generation[0..12]: [185, 187, 189, 186, 189, 189, 174, 184, 189, 173, 189, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229737785Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229760804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[2]R flush_numbers[0..12]: [314, 316, 319, 315, 319, 319, 299, 313, 319, 298, 319, 313]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229779873Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[2]R generation[0..12]: [185, 187, 189, 186, 189, 189, 174, 184, 189, 173, 189, 184]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229798603Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229817902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229833981Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229850671Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:06:20.22986649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229889689Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"":"downstairs","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229906498Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229921848Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"0","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229938487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"1","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229955337Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"client":"2","":"downstairs","session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229972646Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.229988855Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"e59e40eb-8e23-4535-a935-cb4d2385650f is now active with session: 1539e30a-d4e4-43ae-b072-cea0bb242f7d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.230006905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.230024084Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975,"session_id":"1539e30a-d4e4-43ae-b072-cea0bb242f7d"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.230040144Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} {"msg":"Activated sub_volume e59e40eb-8e23-4535-a935-cb4d2385650f","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:20.230053853Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 40960 data: 11 9 7 10 12 14 14 15 17 20 02/30 Flush 03/30 Write block 492 len 12288 data: 17 15 13 04/30 Write block 110 len 12288 data: 8 11 13 05/30 Write block 289 len 12288 data: 14 14 15 06/30 Read block 546 len 32768 07/30 Read block 481 len 28672 08/30 Read block 232 len 36864 09/30 Read block 476 len 12288 10/30 Write block 66 len 32768 data: 13 14 14 13 13 11 14 15 11/30 Read block 300 len 32768 12/30 Write block 175 len 40960 data: 16 15 12 13 12 8 6 7 6 7 13/30 Write block 192 len 24576 data: 11 9 11 13 13 14 14/30 Write block 136 len 24576 data: 11 14 15 13 12 13 15/30 Write block 403 len 8192 data: 18 17 16/30 Write block 544 len 12288 data: 15 12 13 17/30 Read block 169 len 28672 18/30 Write block 320 len 28672 data: 14 15 14 12 8 10 10 19/30 Read block 223 len 24576 20/30 Write block 69 len 4096 data: 14 21/30 Read block 38 len 28672 22/30 Write block 307 len 16384 data: 18 16 19 16 23/30 Write block 87 len 12288 data: 17 16 16 24/30 Write block 23 len 32768 data: 16 19 18 17 15 17 13 12 25/30 Read block 274 len 12288 26/30 Write block 85 len 40960 data: 20 20 18 17 17 12 11 11 10 9 27/30 Write block 421 len 28672 data: 17 19 17 22 19 19 24 28/30 Write block 239 len 36864 data: 14 10 10 13 12 9 10 11 9 29/30 Read block 134 len 12288 30/30 Write block 360 len 36864 data: 8 10 11 12 11 14 13 13 18 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 3 27 27 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:20.24929677Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1975} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 187 187 187 316 316 316 F T T <--- 2 040-059 189 189 189 319 319 319 F F F 3 060-079 186 186 186 315 315 315 F T T <--- 4 080-099 189 189 189 319 319 319 F T T <--- 5 100-119 190 190 190 320 320 320 T T T 6 120-139 174 174 174 299 299 299 F T T <--- 7 140-159 184 184 184 313 313 313 F T T <--- 8 160-179 189 189 189 319 319 319 F T T <--- 9 180-199 173 173 173 298 298 298 F T T <--- 10 200-219 189 189 189 319 319 319 F F F 11 220-239 184 184 184 313 313 313 F T T <--- 12 240-259 186 186 186 315 315 315 F T T <--- 13 260-279 189 189 189 319 319 319 F F F 14 280-299 185 185 185 314 314 314 T T T 15 300-319 185 185 185 314 314 314 F T T <--- 16 320-339 188 188 188 318 318 318 F T T <--- 17 340-359 189 189 189 319 319 319 F F F 18 360-379 189 189 189 319 319 319 F T T <--- 19 380-399 187 187 187 316 316 316 F F F 20 400-419 189 189 189 319 319 319 F T T <--- 21 420-439 189 189 189 319 319 319 F T T <--- 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 172 172 172 297 297 297 T T T 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 189 189 189 319 319 319 F T T <--- 28 560-579 189 189 189 319 319 319 F F F 29 580-599 189 189 189 319 319 319 F F F Max gen: 190, Max flush: 320 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 F F F 1 020-039 191 191 191 321 321 321 F F F 2 040-059 189 189 189 319 319 319 F F F 3 060-079 191 191 191 321 321 321 F F F 4 080-099 191 191 191 321 321 321 F F F 5 100-119 191 191 191 321 321 321 F F F 6 120-139 191 191 191 321 321 321 F F F 7 140-159 191 191 191 321 321 321 F F F 8 160-179 191 191 191 321 321 321 F F F 9 180-199 191 191 191 321 321 321 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 191 191 191 321 321 321 F F F 12 240-259 191 191 191 321 321 321 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 191 191 191 321 321 321 F F F 15 300-319 191 191 191 321 321 321 F F F 16 320-339 191 191 191 321 321 321 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 191 191 191 321 321 321 F F F 19 380-399 187 187 187 316 316 316 F F F 20 400-419 191 191 191 321 321 321 F F F 21 420-439 191 191 191 321 321 321 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 191 191 191 321 321 321 F F F 25 500-519 183 183 183 311 311 311 F F F 26 520-539 181 181 181 307 307 307 F F F 27 540-559 191 191 191 321 321 321 F F F 28 560-579 189 189 189 319 319 319 F F F 29 580-599 189 189 189 319 319 319 F F F Max gen: 191, Max flush: 321 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.729515315Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.773264118Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.773953162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.77401373Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774042038Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774070397Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} {"msg":"Crucible 451b27ba-3d28-4244-8ad5-3492face6179 has session id: 48dbc4f5-b0da-4c0e-9a81-89db44ac7185","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774586828Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Upstairs opts: Upstairs UUID: 451b27ba-3d28-4244-8ad5-3492face6179, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774652245Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Crucible stats registered with UUID: 451b27ba-3d28-4244-8ad5-3492face6179","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774686034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:24.774711453Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774877147Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"1","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774913635Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"2","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.774932035Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"0","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775164946Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775200384Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} {"msg":"451b27ba-3d28-4244-8ad5-3492face6179 active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775279271Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.77529919Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"0","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.77532258Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"1","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775351648Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"2","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775383287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"2","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775416476Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"0","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775448985Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"1","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"ds_connection connected from Ok(127.0.0.1:40577)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775481804Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"2","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"ds_connection connected from Ok(127.0.0.1:40617)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775547901Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"1","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"ds_connection connected from Ok(127.0.0.1:57296)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.775682306Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"io task","client":"0","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.776849602Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"0","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.776949198Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"2","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.777503157Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.777532265Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.778535207Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"1","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[0]R flush_numbers[0..12]: [314, 321, 319, 321, 321, 321, 321, 321, 321, 321, 319, 321]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779076047Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[0]R generation[0..12]: [185, 191, 189, 191, 191, 191, 191, 191, 191, 191, 189, 191]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779105406Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779125835Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[1]R flush_numbers[0..12]: [314, 321, 319, 321, 321, 321, 321, 321, 321, 321, 319, 321]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779147014Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[1]R generation[0..12]: [185, 191, 189, 191, 191, 191, 191, 191, 191, 191, 189, 191]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779177933Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779212421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[2]R flush_numbers[0..12]: [314, 321, 319, 321, 321, 321, 321, 321, 321, 321, 319, 321]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779238851Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[2]R generation[0..12]: [185, 191, 189, 191, 191, 191, 191, 191, 191, 191, 189, 191]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779271619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779301238Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779326207Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779367376Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779395794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779412914Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"":"downstairs","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779432243Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779458002Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"0","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779486751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"1","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.77951625Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"client":"2","":"downstairs","session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779547649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779572898Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"451b27ba-3d28-4244-8ad5-3492face6179 is now active with session: 48dbc4f5-b0da-4c0e-9a81-89db44ac7185","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779591417Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779615546Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 48dbc4f5-b0da-4c0e-9a81-89db44ac7185"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779645005Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} {"msg":"Activated sub_volume 451b27ba-3d28-4244-8ad5-3492face6179","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:24.779667034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 536 len 12288 data: 17 17 16 02/30 Flush 03/30 Write block 489 len 8192 data: 20 20 04/30 Write block 28 len 16384 data: 18 14 13 11 05/30 Read block 51 len 40960 06/30 Read block 344 len 40960 07/30 Write block 529 len 12288 data: 11 13 12 08/30 Read block 354 len 16384 09/30 Write block 529 len 8192 data: 12 14 10/30 Read block 304 len 20480 11/30 Read block 419 len 40960 12/30 Read block 430 len 24576 13/30 Read block 531 len 36864 14/30 Write block 518 len 32768 data: 19 18 18 15 17 16 14 14 15/30 Read block 296 len 4096 16/30 Read block 590 len 28672 17/30 Write block 174 len 4096 data: 16 18/30 Read block 124 len 24576 19/30 Write block 500 len 4096 data: 12 20/30 Read block 491 len 32768 21/30 Read block 407 len 24576 22/30 Read block 468 len 28672 23/30 Read block 478 len 24576 24/30 Flush 25/30 Write block 525 len 36864 data: 15 11 9 8 13 15 13 13 13 26/30 Write block 123 len 36864 data: 19 17 15 15 15 17 11 11 11 27/30 Write block 8 len 12288 data: 17 17 13 28/30 Write block 27 len 8192 data: 16 19 29/30 Read block 259 len 24576 30/30 Read block 437 len 20480 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1001)) Some(JobId(1023)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:24.798576553Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1983} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 314 314 314 T F T <--- 1 020-039 192 191 192 323 321 323 T F T <--- 2 040-059 189 189 189 319 319 319 F F F 3 060-079 191 191 191 321 321 321 F F F 4 080-099 191 191 191 321 321 321 F F F 5 100-119 191 191 191 321 321 321 F F F 6 120-139 191 191 191 321 321 321 T F T <--- 7 140-159 191 191 191 321 321 321 F F F 8 160-179 192 191 192 323 321 323 F F F <--- 9 180-199 191 191 191 321 321 321 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 191 191 191 321 321 321 F F F 12 240-259 191 191 191 321 321 321 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 191 191 191 321 321 321 F F F 15 300-319 191 191 191 321 321 321 F F F 16 320-339 191 191 191 321 321 321 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 191 191 191 321 321 321 F F F 19 380-399 187 187 187 316 316 316 F F F 20 400-419 191 191 191 321 321 321 F F F 21 420-439 191 191 191 321 321 321 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 192 191 192 323 321 323 F F F <--- 25 500-519 192 183 192 323 311 323 F F F <--- 26 520-539 192 192 192 323 322 323 T F T <--- 27 540-559 191 191 191 321 321 321 F F F 28 560-579 189 189 189 319 319 319 F F F 29 580-599 189 189 189 319 319 319 F F F Max gen: 192, Max flush: 323 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 324 324 324 F F F 1 020-039 193 193 193 324 324 324 F F F 2 040-059 189 189 189 319 319 319 F F F 3 060-079 191 191 191 321 321 321 F F F 4 080-099 191 191 191 321 321 321 F F F 5 100-119 191 191 191 321 321 321 F F F 6 120-139 193 193 193 324 324 324 F F F 7 140-159 191 191 191 321 321 321 F F F 8 160-179 192 192 192 323 323 323 F F F 9 180-199 191 191 191 321 321 321 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 191 191 191 321 321 321 F F F 12 240-259 191 191 191 321 321 321 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 191 191 191 321 321 321 F F F 15 300-319 191 191 191 321 321 321 F F F 16 320-339 191 191 191 321 321 321 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 191 191 191 321 321 321 F F F 19 380-399 187 187 187 316 316 316 F F F 20 400-419 191 191 191 321 321 321 F F F 21 420-439 191 191 191 321 321 321 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 192 192 192 323 323 323 F F F 25 500-519 192 192 192 323 323 323 F F F 26 520-539 193 193 193 324 324 324 F F F 27 540-559 191 191 191 321 321 321 F F F 28 560-579 189 189 189 319 319 319 F F F 29 580-599 189 189 189 319 319 319 F F F Max gen: 193, Max flush: 324 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.206598589Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.25093699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.251679691Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.2517287Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.251748849Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.251771538Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} {"msg":"Crucible 4048a865-12f6-4105-a920-aed2f9f1835b has session id: 0b9a9123-9f48-4685-ac65-ab9dc8b61b73","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252372105Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Upstairs opts: Upstairs UUID: 4048a865-12f6-4105-a920-aed2f9f1835b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252425743Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Crucible stats registered with UUID: 4048a865-12f6-4105-a920-aed2f9f1835b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252455882Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:29.252482191Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252677293Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252718882Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252738581Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252926434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.252949033Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} {"msg":"4048a865-12f6-4105-a920-aed2f9f1835b active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.25302699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253049619Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"0","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253071109Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"1","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253100947Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"2","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253138596Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253175134Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253206713Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"ds_connection connected from Ok(127.0.0.1:42461)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253241762Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"ds_connection connected from Ok(127.0.0.1:53540)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253276861Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"ds_connection connected from Ok(127.0.0.1:45495)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.253313269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.254419367Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"1","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.254549942Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"0","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.254989585Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.255095781Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.255363111Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"2","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 319, 321, 321, 321, 324, 321, 323, 321, 319, 321]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.25590363Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[0]R generation[0..12]: [193, 193, 189, 191, 191, 191, 193, 191, 192, 191, 189, 191]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.255933469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.255955019Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 319, 321, 321, 321, 324, 321, 323, 321, 319, 321]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.255974268Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[1]R generation[0..12]: [193, 193, 189, 191, 191, 191, 193, 191, 192, 191, 189, 191]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.255996787Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256016726Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 319, 321, 321, 321, 324, 321, 323, 321, 319, 321]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256035685Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[2]R generation[0..12]: [193, 193, 189, 191, 191, 191, 193, 191, 192, 191, 189, 191]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256054445Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256073984Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256090414Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-02-15T00:06:29.256107483Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256130962Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256146971Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"":"downstairs","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256163771Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.25617898Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"0","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256195949Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"1","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256212639Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"client":"2","":"downstairs","session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256229658Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256247908Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"4048a865-12f6-4105-a920-aed2f9f1835b is now active with session: 0b9a9123-9f48-4685-ac65-ab9dc8b61b73","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256265867Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256281026Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991,"session_id":"0b9a9123-9f48-4685-ac65-ab9dc8b61b73"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256297006Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} {"msg":"Activated sub_volume 4048a865-12f6-4105-a920-aed2f9f1835b","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:29.256310785Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 363 len 28672 data: 13 12 15 14 14 19 13 02/30 Write block 289 len 20480 data: 15 15 16 15 16 03/30 Write block 41 len 32768 data: 15 14 15 13 11 12 16 17 04/30 Read block 537 len 20480 05/30 Write block 517 len 24576 data: 20 20 19 19 16 18 06/30 Write block 102 len 36864 data: 18 18 17 15 13 13 12 10 9 07/30 Read block 129 len 8192 08/30 Read block 396 len 36864 09/30 Flush 10/30 Write block 186 len 36864 data: 11 10 12 13 11 13 12 10 12 11/30 Read block 269 len 8192 12/30 Read block 531 len 32768 13/30 Write block 365 len 36864 data: 16 15 15 20 14 13 11 14 13 14/30 Flush 15/30 Write block 80 len 40960 data: 13 11 14 18 17 21 21 19 18 18 16/30 Read block 522 len 24576 17/30 Write block 396 len 20480 data: 19 20 19 17 15 18/30 Read block 218 len 16384 19/30 Write block 179 len 24576 data: 13 9 7 8 7 8 20/30 Write block 534 len 24576 data: 16 15 18 18 17 15 21/30 Write block 382 len 24576 data: 11 10 10 12 11 9 22/30 Read block 167 len 36864 23/30 Write block 371 len 36864 data: 12 15 14 11 13 12 11 12 10 24/30 Flush 25/30 Write block 237 len 12288 data: 15 13 15 26/30 Read block 480 len 8192 27/30 Read block 162 len 12288 28/30 Read block 200 len 8192 29/30 Write block 579 len 12288 data: 16 17 15 30/30 Read block 93 len 4096 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 30 30 4 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:29.276105241Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1991} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 324 324 324 F F F 1 020-039 193 193 193 324 324 324 F F F 2 040-059 194 194 189 325 325 319 F F T <--- 3 060-079 191 191 191 321 321 321 F F F 4 080-099 194 194 191 327 327 321 F F F <--- 5 100-119 194 194 191 325 325 321 F F F <--- 6 120-139 193 193 193 324 324 324 F F F 7 140-159 191 191 191 321 321 321 F F F 8 160-179 194 194 192 327 327 323 F F F <--- 9 180-199 194 194 191 327 327 321 F F F <--- 10 200-219 189 189 189 319 319 319 F F F 11 220-239 191 191 191 321 321 321 T T F <--- 12 240-259 191 191 191 321 321 321 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 194 194 191 325 325 321 F F T <--- 15 300-319 191 191 191 321 321 321 F F F 16 320-339 191 191 191 321 321 321 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 194 194 191 327 327 321 F F T <--- 19 380-399 194 194 187 327 327 316 F F F <--- 20 400-419 194 194 191 327 327 321 F F F <--- 21 420-439 191 191 191 321 321 321 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 192 192 192 323 323 323 F F F 25 500-519 194 194 192 325 325 323 F F F <--- 26 520-539 194 194 193 327 327 324 F F F <--- 27 540-559 191 191 191 321 321 321 F F F 28 560-579 189 189 189 319 319 319 T T F <--- 29 580-599 189 189 189 319 319 319 T T F <--- Max gen: 194, Max flush: 327 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 324 324 324 F F F 1 020-039 193 193 193 324 324 324 F F F 2 040-059 194 194 194 325 325 325 F F F 3 060-079 191 191 191 321 321 321 F F F 4 080-099 194 194 194 327 327 327 F F F 5 100-119 194 194 194 325 325 325 F F F 6 120-139 193 193 193 324 324 324 F F F 7 140-159 191 191 191 321 321 321 F F F 8 160-179 194 194 194 327 327 327 F F F 9 180-199 194 194 194 327 327 327 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 195 195 195 328 328 328 F F F 12 240-259 191 191 191 321 321 321 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 194 194 194 325 325 325 F F F 15 300-319 191 191 191 321 321 321 F F F 16 320-339 191 191 191 321 321 321 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 194 194 194 327 327 327 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 194 194 194 327 327 327 F F F 21 420-439 191 191 191 321 321 321 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F F F 24 480-499 192 192 192 323 323 323 F F F 25 500-519 194 194 194 325 325 325 F F F 26 520-539 194 194 194 327 327 327 F F F 27 540-559 191 191 191 321 321 321 F F F 28 560-579 195 195 195 328 328 328 F F F 29 580-599 195 195 195 328 328 328 F F F Max gen: 195, Max flush: 328 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.685303572Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:33.728696269Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.729082374Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.760211328Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.760853574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.760911742Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.760931591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.7609595Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} {"msg":"Crucible d379582a-a864-49b1-8b92-2911548d0a9d has session id: 1154e77e-7591-4fec-b843-485d6cd0bb58","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.761593296Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Upstairs opts: Upstairs UUID: d379582a-a864-49b1-8b92-2911548d0a9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.761641434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Crucible stats registered with UUID: d379582a-a864-49b1-8b92-2911548d0a9d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.761666613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:33.761683272Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.761850046Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"0","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.761888555Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"1","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.761917143Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"2","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762065068Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762092997Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} {"msg":"d379582a-a864-49b1-8b92-2911548d0a9d active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762159504Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762180953Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"0","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762198133Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"1","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762214622Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"2","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762230961Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"2","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762247931Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"1","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.76226477Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"0","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:34886)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762358806Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"2","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:42061)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.762439823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"1","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:45793)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.76253387Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"io task","client":"0","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.763648487Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"1","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.763683156Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"0","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.763704355Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"2","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764230905Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764260374Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 325, 321, 327, 325, 324, 321, 327, 327, 319, 328]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764284833Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[0]R generation[0..12]: [193, 193, 194, 191, 194, 194, 193, 191, 194, 194, 189, 195]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764305492Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764325382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 325, 321, 327, 325, 324, 321, 327, 327, 319, 328]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764344201Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[1]R generation[0..12]: [193, 193, 194, 191, 194, 194, 193, 191, 194, 194, 189, 195]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.76436279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764382449Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 325, 321, 327, 325, 324, 321, 327, 327, 319, 328]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764401319Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[2]R generation[0..12]: [193, 193, 194, 191, 194, 194, 193, 191, 194, 194, 189, 195]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764420238Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764443587Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764460317Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764477216Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764493165Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764509125Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"":"downstairs","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:06:33.764525794Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764545503Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"0","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764562402Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"1","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764578902Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"client":"2","":"downstairs","session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764595751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764611591Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"d379582a-a864-49b1-8b92-2911548d0a9d is now active with session: 1154e77e-7591-4fec-b843-485d6cd0bb58","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.76462937Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764644399Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999,"session_id":"1154e77e-7591-4fec-b843-485d6cd0bb58"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764660279Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} {"msg":"Activated sub_volume d379582a-a864-49b1-8b92-2911548d0a9d","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:33.764674088Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 323 len 12288 data: 13 9 11 02/30 Write block 68 len 20480 data: 15 15 14 12 15 03/30 Write block 562 len 32768 data: 18 19 16 11 9 10 12 14 04/30 Read block 226 len 16384 05/30 Write block 428 len 36864 data: 22 18 16 19 15 14 18 20 19 06/30 Read block 436 len 24576 07/30 Read block 551 len 12288 08/30 Write block 496 len 36864 data: 14 11 11 10 13 11 15 18 17 09/30 Read block 95 len 36864 10/30 Read block 225 len 40960 11/30 Flush 12/30 Write block 180 len 32768 data: 10 8 9 8 9 11 12 11 13/30 Write block 304 len 28672 data: 17 19 18 19 17 20 17 14/30 Write block 376 len 12288 data: 13 12 13 15/30 Flush 16/30 Write block 494 len 32768 data: 14 13 15 12 12 11 14 12 17/30 Write block 255 len 16384 data: 13 12 11 14 18/30 Read block 582 len 32768 19/30 Read block 589 len 4096 20/30 Write block 564 len 16384 data: 17 12 10 11 21/30 Write block 29 len 8192 data: 15 14 22/30 Write block 55 len 8192 data: 15 13 23/30 Write block 174 len 28672 data: 17 17 16 13 14 14 11 24/30 Read block 151 len 20480 25/30 Write block 107 len 8192 data: 14 13 26/30 Read block 180 len 36864 27/30 Read block 479 len 4096 28/30 Read block 4 len 28672 29/30 Write block 291 len 8192 data: 17 16 30/30 Write block 473 len 20480 data: 10 10 7 8 8 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 5 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 2 32 Done 1 29 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:33.783535199Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":1999} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 324 324 324 F F F 1 020-039 193 193 193 324 324 324 F T T <--- 2 040-059 194 194 194 325 325 325 F T T <--- 3 060-079 191 196 196 321 329 329 F F F <--- 4 080-099 194 194 194 327 327 327 F F F 5 100-119 194 194 194 325 325 325 F T T <--- 6 120-139 193 193 193 324 324 324 F F F 7 140-159 191 191 191 321 321 321 F F F 8 160-179 194 194 194 327 327 327 F T T <--- 9 180-199 194 196 196 327 330 330 F T T <--- 10 200-219 189 189 189 319 319 319 F F F 11 220-239 195 195 195 328 328 328 F F F 12 240-259 191 191 191 321 321 321 F T T <--- 13 260-279 189 189 189 319 319 319 F F F 14 280-299 194 194 194 325 325 325 F T T <--- 15 300-319 191 196 196 321 330 330 F F F <--- 16 320-339 191 196 196 321 329 329 T F F <--- 17 340-359 189 189 189 319 319 319 F F F 18 360-379 194 196 196 327 330 330 F F F <--- 19 380-399 194 194 194 327 327 327 F F F 20 400-419 194 194 194 327 327 327 F F F 21 420-439 191 196 196 321 329 329 F F F <--- 22 440-459 188 188 188 317 317 317 F F F 23 460-479 176 176 176 303 303 303 F T T <--- 24 480-499 192 196 196 323 329 329 F T T <--- 25 500-519 194 196 196 325 329 329 F T T <--- 26 520-539 194 194 194 327 327 327 F F F 27 540-559 191 191 191 321 321 321 F F F 28 560-579 195 196 196 328 329 329 F T T <--- 29 580-599 195 195 195 328 328 328 F F F Max gen: 196, Max flush: 330 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 324 324 324 F F F 1 020-039 197 197 197 331 331 331 F F F 2 040-059 197 197 197 331 331 331 F F F 3 060-079 196 196 196 329 329 329 F F F 4 080-099 194 194 194 327 327 327 F F F 5 100-119 197 197 197 331 331 331 F F F 6 120-139 193 193 193 324 324 324 F F F 7 140-159 191 191 191 321 321 321 F F F 8 160-179 197 197 197 331 331 331 F F F 9 180-199 197 197 197 331 331 331 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 195 195 195 328 328 328 F F F 12 240-259 197 197 197 331 331 331 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 197 197 197 331 331 331 F F F 15 300-319 196 196 196 330 330 330 F F F 16 320-339 196 196 196 329 329 329 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 194 194 194 327 327 327 F F F 21 420-439 196 196 196 329 329 329 F F F 22 440-459 188 188 188 317 317 317 F F F 23 460-479 197 197 197 331 331 331 F F F 24 480-499 197 197 197 331 331 331 F F F 25 500-519 197 197 197 331 331 331 F F F 26 520-539 194 194 194 327 327 327 F F F 27 540-559 191 191 191 321 321 321 F F F 28 560-579 197 197 197 331 331 331 F F F 29 580-599 195 195 195 328 328 328 F F F Max gen: 197, Max flush: 331 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.24024579Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.284591251Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.285301103Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"59356dbe60c15a36ffe06f282184f10210d934e9\",\n git_commit_timestamp: \"2026-02-14T23:34:14.000000000Z\",\n git_branch: \"alan/cmon-polish\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.285360821Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.28538104Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.285408459Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} {"msg":"Crucible af0a3dca-3c86-4bbe-afae-2dd8735dedfa has session id: 054d0c94-9773-423b-a6fe-e05319b3c3cd","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.285935469Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Upstairs opts: Upstairs UUID: af0a3dca-3c86-4bbe-afae-2dd8735dedfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286018086Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Crucible stats registered with UUID: af0a3dca-3c86-4bbe-afae-2dd8735dedfa","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286059124Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:38.286089823Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286284746Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"1","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286337364Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"0","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286359663Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"2","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286501488Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286539246Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} {"msg":"af0a3dca-3c86-4bbe-afae-2dd8735dedfa active request set","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286599034Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286637382Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"0","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286671281Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"1","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.28670524Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"2","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286738759Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"2","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286769628Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"0","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286797606Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"1","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60880)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286826425Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"2","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58264)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286856684Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"0","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:56372)","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.286992649Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"io task","client":"1","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.288123866Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"0","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.28827209Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"1","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.288750492Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.288837479Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.288876217Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"2","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[0]R flush_numbers[0..12]: [324, 331, 331, 329, 327, 331, 324, 321, 331, 331, 319, 328]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289424036Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[0]R generation[0..12]: [193, 197, 197, 196, 194, 197, 193, 191, 197, 197, 189, 195]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289454665Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289475434Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[1]R flush_numbers[0..12]: [324, 331, 331, 329, 327, 331, 324, 321, 331, 331, 319, 328]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289494614Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[1]R generation[0..12]: [193, 197, 197, 196, 194, 197, 193, 191, 197, 197, 189, 195]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289513573Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289533072Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[2]R flush_numbers[0..12]: [324, 331, 331, 329, 327, 331, 324, 321, 331, 331, 319, 328]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289551782Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[2]R generation[0..12]: [193, 197, 197, 196, 194, 197, 193, 191, 197, 197, 189, 195]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289570621Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.28959024Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289606699Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-02-15T00:06:38.289623819Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289644548Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289660657Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"":"downstairs","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289677437Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289708996Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"0","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289727145Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"1","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289743734Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"client":"2","":"downstairs","session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289760574Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289776613Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"af0a3dca-3c86-4bbe-afae-2dd8735dedfa is now active with session: 054d0c94-9773-423b-a6fe-e05319b3c3cd","v":0,"nameDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"crucible","level":30,"time":"2026-02-15T00:06:38.289798162Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289813421Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007,"session_id":"054d0c94-9773-423b-a6fe-e05319b3c3cd"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.289829751Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} {"msg":"Activated sub_volume af0a3dca-3c86-4bbe-afae-2dd8735dedfa","v":0,"name":"crucible","level":30,"time":"2026-02-15T00:06:38.28984377Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 533 len 40960 data: 14 17 16 19 19 18 16 12 13 13 02/30 Read block 419 len 36864 03/30 Write block 84 len 24576 data: 18 22 22 20 19 19 04/30 Read block 241 len 24576 05/30 Read block 114 len 20480 06/30 Flush 07/30 Write block 282 len 36864 data: 11 12 10 12 12 14 13 16 16 08/30 Write block 129 len 8192 data: 12 12 09/30 Read block 120 len 24576 10/30 Write block 158 len 16384 data: 12 16 15 16 11/30 Read block 297 len 20480 12/30 Write block 119 len 24576 data: 16 16 16 17 20 18 13/30 Read block 162 len 28672 14/30 Read block 210 len 36864 15/30 Write block 99 len 36864 data: 12 17 18 19 19 18 16 14 15 16/30 Write block 446 len 24576 data: 17 16 15 15 12 13 17/30 Read block 412 len 4096 18/30 Flush 19/30 Write block 400 len 32768 data: 16 13 14 19 18 18 18 15 20/30 Write block 180 len 12288 data: 12 9 10 21/30 Write block 522 len 4096 data: 19 22/30 Read block 315 len 40960 23/30 Read block 23 len 24576 24/30 Write block 14 len 36864 data: 14 13 14 15 14 15 16 20 16 25/30 Write block 110 len 20480 data: 10 12 14 15 15 26/30 Write block 86 len 8192 data: 23 21 27/30 Write block 230 len 4096 data: 14 28/30 Read block 502 len 16384 29/30 Read block 385 len 40960 30/30 Write block 176 len 20480 data: 17 14 15 15 13 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-02-15T00:06:38.30953444Z","hostname":"w-01KHF98DFGT2D9W4JTH0D44QBA","pid":2007} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 324 324 324 T T F <--- 1 020-039 197 197 197 331 331 331 T T F <--- 2 040-059 197 197 197 331 331 331 F F F 3 060-079 196 196 196 329 329 329 F F F 4 080-099 198 198 194 333 333 327 T T F <--- 5 100-119 198 198 197 333 333 331 T T F <--- 6 120-139 198 198 193 333 333 324 F F F <--- 7 140-159 198 198 191 333 333 321 F F F <--- 8 160-179 198 198 197 333 333 331 T T F <--- 9 180-199 197 197 197 331 331 331 T T F <--- 10 200-219 189 189 189 319 319 319 F F F 11 220-239 195 195 195 328 328 328 T T F <--- 12 240-259 197 197 197 331 331 331 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 198 198 197 333 333 331 F F F <--- 15 300-319 196 196 196 330 330 330 F F F 16 320-339 196 196 196 329 329 329 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 194 194 194 327 327 327 T T F <--- 21 420-439 196 196 196 329 329 329 F F F 22 440-459 198 198 188 333 333 317 F F F <--- 23 460-479 197 197 197 331 331 331 F F F 24 480-499 197 197 197 331 331 331 F F F 25 500-519 197 197 197 331 331 331 F F F 26 520-539 198 198 194 332 332 327 T T T <--- 27 540-559 198 198 191 332 332 321 F F T <--- 28 560-579 197 197 197 331 331 331 F F F 29 580-599 195 195 195 328 328 328 F F F Max gen: 198, Max flush: 333 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 334 334 334 F F F 1 020-039 199 199 199 334 334 334 F F F 2 040-059 197 197 197 331 331 331 F F F 3 060-079 196 196 196 329 329 329 F F F 4 080-099 199 199 199 334 334 334 F F F 5 100-119 199 199 199 334 334 334 F F F 6 120-139 198 198 198 333 333 333 F F F 7 140-159 198 198 198 333 333 333 F F F 8 160-179 199 199 199 334 334 334 F F F 9 180-199 199 199 199 334 334 334 F F F 10 200-219 189 189 189 319 319 319 F F F 11 220-239 199 199 199 334 334 334 F F F 12 240-259 197 197 197 331 331 331 F F F 13 260-279 189 189 189 319 319 319 F F F 14 280-299 198 198 198 333 333 333 F F F 15 300-319 196 196 196 330 330 330 F F F 16 320-339 196 196 196 329 329 329 F F F 17 340-359 189 189 189 319 319 319 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 199 199 199 334 334 334 F F F 21 420-439 196 196 196 329 329 329 F F F 22 440-459 198 198 198 333 333 333 F F F 23 460-479 197 197 197 331 331 331 F F F 24 480-499 197 197 197 331 331 331 F F F 25 500-519 197 197 197 331 331 331 F F F 26 520-539 199 199 199 334 334 334 F F F 27 540-559 198 198 198 332 332 332 F F F 28 560-579 197 197 197 331 331 331 F F F 29 580-599 195 195 195 328 328 328 F F F Max gen: 199, Max flush: 334 10:32 Test duration Test completed real 10:31.815645478 user 59.450496961 sys 25.889145370 trap 0.080968590 tflt 0.094610281 dflt 0.122567048 kflt 0.005944198 lock 16:39:53.980134211 slp 51:10.911830899 lat 6.244557060 stop 0.035115009