Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.1976916 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.013105578 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.011885538 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.738236249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.750799193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.751664796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.751763309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.751773738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.751787568Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"Crucible 718e6f6c-4022-468c-ab0b-75b174a2f3f0 has session id: 5827ea88-c06f-456f-b2de-3c6fc2075760","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.752392068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Upstairs opts: Upstairs UUID: 718e6f6c-4022-468c-ab0b-75b174a2f3f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.752481752Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Crucible stats registered with UUID: 718e6f6c-4022-468c-ab0b-75b174a2f3f0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.752494221Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:20.752544728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.752989169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753056394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753070643Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753229773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753286349Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"718e6f6c-4022-468c-ab0b-75b174a2f3f0 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753326976Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753336446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"0","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753346155Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"1","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753355754Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"2","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753365234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753403541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.75342176Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"ds_connection connected from Ok(127.0.0.1:37336)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753501235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"ds_connection connected from Ok(127.0.0.1:37053)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753512584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"ds_connection connected from Ok(127.0.0.1:48704)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.753523123Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.754925841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"0","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.754951919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"1","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755085461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"2","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755577978Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755590697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755652313Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755666212Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755679282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755692721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.75570484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755718989Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755732098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755744717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755758766Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755770146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755781885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755793994Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755806743Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"":"downstairs","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755819182Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755830342Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"0","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755841981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"1","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.75585348Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"client":"2","":"downstairs","session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755866289Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755877429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"718e6f6c-4022-468c-ab0b-75b174a2f3f0 is now active with session: 5827ea88-c06f-456f-b2de-3c6fc2075760","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755889338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755900377Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"session_id":"5827ea88-c06f-456f-b2de-3c6fc2075760"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755912176Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"Activated sub_volume 718e6f6c-4022-468c-ab0b-75b174a2f3f0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755921906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.755931705Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.756143131Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- ,"time":"2026-09-18T03:28:20.793391291Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 {States:"msg":" Active ActiveCLIENT: Tests done. All submitted work has been ACK'd Active" ,"v":0,"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY name":"crucible","level":30 1004 Acked Read 88 Done, Done" Donetime" false:" 2026-09-18T03:28:20.793433018Z"," 1005 Acked Read 256hostname Done" Done:" Done false w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ", 1006 Acked Read 256" Done Donepid Done" false: 1139,"task":"crutest"} STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:20.793504604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.795081869Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.795147515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:24.795164184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1139} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.818872404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.827261572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.827551413Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.827580501Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.827589101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.82760102Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Crucible b9f83b34-8a6e-451f-99b3-462281551a9a has session id: 5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.827931248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Upstairs opts: Upstairs UUID: b9f83b34-8a6e-451f-99b3-462281551a9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.827987985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Crucible stats registered with UUID: b9f83b34-8a6e-451f-99b3-462281551a9a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828007183Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:24.828019123Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828157594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828172733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828182932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828345991Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.82836966Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"b9f83b34-8a6e-451f-99b3-462281551a9a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828393708Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828403027Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"0","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828412917Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"1","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828422666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"2","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828432385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828442525Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828452544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46167)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828523749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43767)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828625543Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39311)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.828650621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829184486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"0","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.82926931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"2","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829299958Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"1","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829490906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.82958457Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829646846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829657655Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829668724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829679023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829689213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829700192Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829710571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829720701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.82973168Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829744339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829754019Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829763268Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829772707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"":"downstairs","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829782477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829791216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"0","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829800985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"1","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829810515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"client":"2","":"downstairs","session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829820434Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829829684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"b9f83b34-8a6e-451f-99b3-462281551a9a is now active with session: 5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829839703Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829848882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829856372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"session_id":"5bf28a1c-bf99-43d9-ba52-0f7f3801eaa3"} {"msg":"Activated sub_volume b9f83b34-8a6e-451f-99b3-462281551a9a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829865541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.829873641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.830286114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"task":"crutest"} 01/30 Write block 422 len 4096 data: 2 02/30 Write block 467 len 28672 data: 2 2 2 2 2 2 2 03/30 Write block 355 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 545 len 28672 05/30 Flush 06/30 Write block 169 len 28672 data: 2 2 2 2 2 2 2 07/30 Write block 302 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Write block 125 len 28672 data: 2 2 2 2 2 2 2 09/30 Read block 529 len 12288 10/30 Write block 147 len 20480 data: 2 2 2 2 2 11/30 Write block 584 len 32768 data: 2 2 2 2 2 2 2 2 12/30 Read block 61 len 40960 13/30 Read block 259 len 16384 14/30 Flush 15/30 Write block 196 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Flush 17/30 Write block 159 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Write block 583 len 40960 data: 2 3 3 3 3 3 3 3 3 2 19/30 Flush 20/30 Write block 249 len 24576 data: 2 2 2 2 2 2 21/30 Read block 393 len 20480 22/30 Write block 177 len 24576 data: 2 2 2 2 2 2 23/30 Read block 409 len 20480 24/30 Read block 444 len 40960 25/30 Read block 549 len 12288 26/30 Read block 317 len 12288 27/30 Write block 424 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 428 len 32768 29/30 Read block 549 len 32768 30/30 Read block 243 len 8192 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:24.839198117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:24.839354497Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 1 2 3 1 3 F F F <--- 7 140-159 2 1 2 5 1 5 F F F <--- 8 160-179 2 1 2 5 1 5 T F T <--- 9 180-199 2 1 2 4 1 4 T F T <--- 10 200-219 2 1 2 4 1 4 F F F <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 T F T <--- 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 1 2 3 1 3 F F F <--- 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 1 2 2 1 2 F F F <--- 18 360-379 2 1 2 2 1 2 F F F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 1 2 2 1 2 T F T <--- 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 1 2 2 1 2 F F F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 1 2 5 1 5 F F F <--- Max gen: 2, Max flush: 5 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 2 2 2 5 5 5 F F F 8 160-179 3 3 3 6 6 6 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 2 2 2 4 4 4 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 5 5 5 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.223400726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.231608036Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.231889158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.231924625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.231933825Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.231945784Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Crucible 5fc01f89-31c4-4db7-b5de-9b3e578dcfb9 has session id: 4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232271433Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Upstairs opts: Upstairs UUID: 5fc01f89-31c4-4db7-b5de-9b3e578dcfb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232290191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Crucible stats registered with UUID: 5fc01f89-31c4-4db7-b5de-9b3e578dcfb9","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232300281Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:29.23230955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232449461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232478909Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232489658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.23262076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232636818Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"5fc01f89-31c4-4db7-b5de-9b3e578dcfb9 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232660827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232672516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"0","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232682415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"1","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232692145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"2","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232701784Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232711884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232721783Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"ds_connection connected from Ok(127.0.0.1:56079)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232757961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"ds_connection connected from Ok(127.0.0.1:56671)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.23276906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"ds_connection connected from Ok(127.0.0.1:48171)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.232804617Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233385969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"2","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233405258Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"0","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233416847Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"1","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233691589Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233714058Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 3, 5, 6, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233759625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233774104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233785393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 3, 5, 6, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233795862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233805872Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233817001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 3, 5, 6, 6, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.23382711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.23383747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233848429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233857918Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233867548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233876967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233886306Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"":"downstairs","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233896336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233905125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"0","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233915144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"1","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233926574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"client":"2","":"downstairs","session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233936623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233945862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"5fc01f89-31c4-4db7-b5de-9b3e578dcfb9 is now active with session: 4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233955942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233964591Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"session_id":"4529e2c3-d5fd-4c40-8bf8-8bb5b3b29f10"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233973671Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Activated sub_volume 5fc01f89-31c4-4db7-b5de-9b3e578dcfb9","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.2339817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.233990069Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.234236063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"task":"crutest"} 01/30 Flush 02/30 Write block 403 len 4096 data: 2 03/30 Read block 178 len 28672 04/30 Read block 14 len 8192 05/30 Read block 344 len 4096 06/30 Read block 388 len 20480 07/30 Read block 174 len 36864 08/30 Write block 304 len 4096 data: 3 09/30 Write block 347 len 4096 data: 2 10/30 Write block 44 len 28672 data: 2 2 2 2 2 2 2 11/30 Write block 391 len 20480 data: 2 2 2 2 2 12/30 Read block 570 len 32768 13/30 Write block 566 len 28672 data: 2 2 2 2 2 2 2 14/30 Write block 166 len 36864 data: 3 3 2 3 3 3 3 3 3 15/30 Read block 401 len 20480 16/30 Read block 484 len 40960 17/30 Read block 79 len 36864 18/30 Read block 226 len 20480 19/30 Read block 94 len 20480 20/30 Write block 126 len 24576 data: 3 3 3 3 3 3 21/30 Write block 102 len 8192 data: 2 2 22/30 Write block 210 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 574 len 28672 24/30 Write block 366 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Write block 555 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Read block 211 len 36864 27/30 Read block 314 len 24576 28/30 Write block 285 len 8192 data: 2 2 29/30 Write block 512 len 28672 data: 2 2 2 2 2 2 2 30/30 Write block 409 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 27 3 27 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:29.241907719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:29.242101226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T F T <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 T F T <--- 6 120-139 2 2 2 3 3 3 T F T <--- 7 140-159 2 2 2 5 5 5 F F F 8 160-179 3 3 3 6 6 6 T F T <--- 9 180-199 3 3 3 6 6 6 F F F 10 200-219 2 2 2 4 4 4 T F T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 T F T <--- 15 300-319 2 2 2 3 3 3 T F T <--- 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 2 2 T F T <--- 18 360-379 2 2 2 2 2 2 T F T <--- 19 380-399 1 1 1 1 1 1 T F T <--- 20 400-419 1 1 1 1 1 1 T T T 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 T F T <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 T F T <--- 28 560-579 1 1 1 1 1 1 T F T <--- 29 580-599 2 2 2 5 5 5 F F F Max gen: 3, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 5 5 5 F F F 8 160-179 4 4 4 7 7 7 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 6 6 6 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 4 4 4 7 7 7 F F F 20 400-419 4 4 4 7 7 7 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 4 4 4 7 7 7 F F F 29 580-599 2 2 2 5 5 5 F F F Max gen: 4, Max flush: 7 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.494748778Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50292166Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503196352Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50322232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50323075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503242649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Crucible 5c6f00a2-d154-4e94-ae3e-8178ed29dd4f has session id: 6f95249a-4e56-423d-857e-2413b9552217","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503593936Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Upstairs opts: Upstairs UUID: 5c6f00a2-d154-4e94-ae3e-8178ed29dd4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503626404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Crucible stats registered with UUID: 5c6f00a2-d154-4e94-ae3e-8178ed29dd4f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503637593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:33.503647082Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503789523Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503818351Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50382946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503958452Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503976901Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"5c6f00a2-d154-4e94-ae3e-8178ed29dd4f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.503993669Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504002709Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"0","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504012678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"1","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504022348Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"2","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504031957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504041956Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504052026Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"ds_connection connected from Ok(127.0.0.1:58550)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504093733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"ds_connection connected from Ok(127.0.0.1:33770)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504106142Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"ds_connection connected from Ok(127.0.0.1:35741)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504189767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50474859Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"0","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504776988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"1","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.504889031Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"2","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50505662Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505077718Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 7, 1, 1, 7, 7, 5, 7, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505183931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[0]R generation[0..12]: [1, 1, 4, 1, 1, 4, 4, 2, 4, 3, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50519513Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50520642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 7, 1, 1, 7, 7, 5, 7, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505216829Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[1]R generation[0..12]: [1, 1, 4, 1, 1, 4, 4, 2, 4, 3, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505227158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505238108Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 7, 1, 1, 7, 7, 5, 7, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505248347Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[2]R generation[0..12]: [1, 1, 4, 1, 1, 4, 4, 2, 4, 3, 4, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505261266Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505272455Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Max found gen is 5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505281895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Generation requested: 6 >= found:5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505291674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505301073Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505310573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"":"downstairs","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505320342Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505329212Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"0","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505338991Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"1","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.50534874Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"client":"2","":"downstairs","session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.5053586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505368129Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"5c6f00a2-d154-4e94-ae3e-8178ed29dd4f is now active with session: 6f95249a-4e56-423d-857e-2413b9552217","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505378198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505387098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"session_id":"6f95249a-4e56-423d-857e-2413b9552217"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505396397Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Activated sub_volume 5c6f00a2-d154-4e94-ae3e-8178ed29dd4f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505404527Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505414326Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.505726096Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"task":"crutest"} 01/30 Write block 404 len 40960 data: 2 2 2 2 2 3 3 3 3 2 02/30 Write block 205 len 36864 data: 3 2 2 2 2 3 3 3 3 03/30 Write block 197 len 40960 data: 3 3 3 3 3 3 3 3 4 3 04/30 Read block 399 len 4096 05/30 Read block 58 len 16384 06/30 Write block 23 len 8192 data: 2 2 07/30 Read block 362 len 24576 08/30 Read block 403 len 16384 09/30 Read block 47 len 8192 10/30 Write block 181 len 40960 data: 3 3 2 2 2 2 2 2 2 2 11/30 Flush 12/30 Write block 274 len 8192 data: 2 2 13/30 Read block 300 len 36864 14/30 Write block 5 len 16384 data: 2 2 2 2 15/30 Read block 484 len 16384 16/30 Read block 221 len 24576 17/30 Write block 56 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Write block 291 len 12288 data: 2 2 2 19/30 Read block 32 len 28672 20/30 Read block 13 len 40960 21/30 Flush 22/30 Write block 213 len 8192 data: 4 3 23/30 Flush 24/30 Write block 51 len 28672 data: 2 2 2 2 2 3 3 25/30 Write block 456 len 24576 data: 2 2 2 2 2 2 26/30 Read block 384 len 24576 27/30 Read block 552 len 40960 28/30 Write block 239 len 16384 data: 2 2 2 2 29/30 Read block 106 len 4096 30/30 Write block 364 len 28672 data: 2 2 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 4 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 26 28 Done 29 29 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:33.514212737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:33.514396325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 1 9 9 1 F F F <--- 1 020-039 6 6 1 8 8 1 F F F <--- 2 040-059 6 6 4 9 9 7 T T F <--- 3 060-079 6 6 1 9 9 1 F F F <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 5 5 5 F F F 8 160-179 4 4 4 7 7 7 F F F 9 180-199 6 6 3 8 8 6 F F T <--- 10 200-219 6 6 4 10 10 7 F F T <--- 11 220-239 1 1 1 1 1 1 T T F <--- 12 240-259 3 3 3 6 6 6 T T F <--- 13 260-279 6 6 1 9 9 1 F F F <--- 14 280-299 6 6 4 9 9 7 F F F <--- 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 7 7 7 T T F <--- 19 380-399 4 4 4 7 7 7 F F F 20 400-419 6 6 4 8 8 7 F F T <--- 21 420-439 3 3 3 6 6 6 F F F 22 440-459 1 1 1 1 1 1 T T F <--- 23 460-479 2 2 2 2 2 2 T T F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 4 4 4 7 7 7 F F F 29 580-599 2 2 2 5 5 5 F F F Max gen: 6, Max flush: 10 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 9 9 9 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 6 6 6 9 9 9 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 5 5 5 F F F 8 160-179 4 4 4 7 7 7 F F F 9 180-199 6 6 6 8 8 8 F F F 10 200-219 6 6 6 10 10 10 F F F 11 220-239 7 7 7 11 11 11 F F F 12 240-259 7 7 7 11 11 11 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 6 6 6 9 9 9 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 11 11 11 F F F 19 380-399 4 4 4 7 7 7 F F F 20 400-419 6 6 6 8 8 8 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 4 4 4 7 7 7 F F F 29 580-599 2 2 2 5 5 5 F F F Max gen: 7, Max flush: 11 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.762224665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:37.769563692Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.769627798Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.776924248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777213109Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777239938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777248517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777260356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Crucible 36cade98-290f-4672-84ed-0fdd9a28c363 has session id: 02d62457-b2a1-4927-8818-76b842de61cf","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777632352Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Upstairs opts: Upstairs UUID: 36cade98-290f-4672-84ed-0fdd9a28c363, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77766336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Crucible stats registered with UUID: 36cade98-290f-4672-84ed-0fdd9a28c363","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777674619Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:37.777683848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77781779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777851307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.777862767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778006767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778016036Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"36cade98-290f-4672-84ed-0fdd9a28c363 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778039985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778049254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"0","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778058894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"1","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778068403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"2","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778077802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778087872Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778097761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:48836)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778165267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58203)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778291438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:50747)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778303788Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778813204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"2","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77887231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"1","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.778999922Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"0","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779139663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779157521Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[0]R flush_numbers[0..12]: [9, 8, 11, 9, 1, 7, 7, 5, 7, 8, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779275864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[0]R generation[0..12]: [6, 6, 7, 6, 1, 4, 4, 2, 4, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779287043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779298312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[1]R flush_numbers[0..12]: [9, 8, 11, 9, 1, 7, 7, 5, 7, 8, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779308811Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[1]R generation[0..12]: [6, 6, 7, 6, 1, 4, 4, 2, 4, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779319181Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77933039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[2]R flush_numbers[0..12]: [9, 8, 11, 9, 1, 7, 7, 5, 7, 8, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779340849Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[2]R generation[0..12]: [6, 6, 7, 6, 1, 4, 4, 2, 4, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779351619Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779362888Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779372377Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779382157Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779393826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779403535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"":"downstairs","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779413565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779422474Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"0","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779432273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"1","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779442173Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"client":"2","":"downstairs","session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779452102Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779461371Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"36cade98-290f-4672-84ed-0fdd9a28c363 is now active with session: 02d62457-b2a1-4927-8818-76b842de61cf","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779471571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77948042Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"session_id":"02d62457-b2a1-4927-8818-76b842de61cf"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77949003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Activated sub_volume 36cade98-290f-4672-84ed-0fdd9a28c363","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779498359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.779506738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.77979091Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} 01/30 Write block 517 len 8192 data: 3 3 02/30 Write block 362 len 12288 data: 2 2 3 03/30 Write block 222 len 8192 data: 2 2 04/30 Read block 1 len 28672 05/30 Write block 187 len 8192 data: 3 3 06/30 Flush 07/30 Write block 98 len 32768 data: 2 2 2 2 3 3 2 2 08/30 Write block 556 len 28672 data: 3 3 3 3 3 3 3 09/30 Write block 212 len 36864 data: 4 5 4 3 3 3 2 2 2 10/30 Write block 493 len 8192 data: 2 2 11/30 Read block 548 len 36864 12/30 Read block 41 len 24576 13/30 Read block 272 len 28672 14/30 Write block 194 len 8192 data: 2 2 15/30 Flush 16/30 Write block 592 len 32768 data: 3 2 2 2 2 2 2 2 17/30 Write block 359 len 4096 data: 3 18/30 Read block 33 len 40960 19/30 Read block 58 len 24576 20/30 Read block 306 len 12288 21/30 Write block 404 len 40960 data: 3 3 3 3 3 4 4 4 4 3 22/30 Write block 124 len 4096 data: 2 23/30 Read block 538 len 4096 24/30 Read block 405 len 36864 25/30 Write block 204 len 36864 data: 4 5 4 3 3 3 4 4 5 26/30 Read block 171 len 20480 27/30 Read block 584 len 24576 28/30 Write block 184 len 4096 data: 3 29/30 Read block 32 len 20480 30/30 Write block 14 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:37.788238284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:37.788409403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 9 9 9 F T T <--- 1 020-039 6 6 6 8 8 8 F T T <--- 2 040-059 7 7 7 11 11 11 F F F 3 060-079 6 6 6 9 9 9 F F F 4 080-099 1 8 8 1 13 13 F F F <--- 5 100-119 4 8 8 7 13 13 F F F <--- 6 120-139 4 4 4 7 7 7 F T T <--- 7 140-159 2 2 2 5 5 5 F F F 8 160-179 4 4 4 7 7 7 F F F 9 180-199 6 8 8 8 13 13 F T T <--- 10 200-219 6 8 8 10 13 13 F T T <--- 11 220-239 7 8 8 11 13 13 F F F <--- 12 240-259 7 7 7 11 11 11 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 6 6 6 9 9 9 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F T T <--- 18 360-379 7 8 8 11 12 12 F F F <--- 19 380-399 4 4 4 7 7 7 F F F 20 400-419 6 6 6 8 8 8 F T T <--- 21 420-439 3 3 3 6 6 6 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 1 8 8 1 13 13 F F F <--- 25 500-519 4 8 8 7 12 12 F F F <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 4 8 8 7 13 13 F F F <--- 28 560-579 4 8 8 7 13 13 F F F <--- 29 580-599 2 2 2 5 5 5 F T T <--- Max gen: 8, Max flush: 13 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 14 14 14 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 6 6 6 9 9 9 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 8 8 8 13 13 13 F F F 6 120-139 9 9 9 14 14 14 F F F 7 140-159 2 2 2 5 5 5 F F F 8 160-179 4 4 4 7 7 7 F F F 9 180-199 9 9 9 14 14 14 F F F 10 200-219 9 9 9 14 14 14 F F F 11 220-239 8 8 8 13 13 13 F F F 12 240-259 7 7 7 11 11 11 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 6 6 6 9 9 9 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 9 9 9 14 14 14 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 4 4 4 7 7 7 F F F 20 400-419 9 9 9 14 14 14 F F F 21 420-439 3 3 3 6 6 6 F F F 22 440-459 7 7 7 11 11 11 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 8 8 8 12 12 12 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 9 9 9 14 14 14 F F F Max gen: 9, Max flush: 14 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.050764195Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059136205Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059411187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059435535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059447414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059459083Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Crucible 228dcb8c-2b2a-4d5c-9c50-02f720f1d448 has session id: 20d9b015-756e-46d7-ac5d-a39ce79cd940","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059789252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Upstairs opts: Upstairs UUID: 228dcb8c-2b2a-4d5c-9c50-02f720f1d448, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.05981857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Crucible stats registered with UUID: 228dcb8c-2b2a-4d5c-9c50-02f720f1d448","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.059829739Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:42.059838898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.05997483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060005787Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060017117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060167017Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060194125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"228dcb8c-2b2a-4d5c-9c50-02f720f1d448 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060209534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060218623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"0","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060228393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"1","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060240642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"2","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060250201Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060260101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060285769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"ds_connection connected from Ok(127.0.0.1:57886)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060303918Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"ds_connection connected from Ok(127.0.0.1:59085)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060353935Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"ds_connection connected from Ok(127.0.0.1:46527)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060438039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.060968384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"2","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061012971Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"1","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061065748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"0","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061312732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061351299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 11, 9, 13, 13, 14, 5, 7, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061428254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[0]R generation[0..12]: [9, 9, 7, 6, 8, 8, 9, 2, 4, 9, 9, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061439233Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061450412Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 11, 9, 13, 13, 14, 5, 7, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061463542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[1]R generation[0..12]: [9, 9, 7, 6, 8, 8, 9, 2, 4, 9, 9, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061473861Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.06148496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 11, 9, 13, 13, 14, 5, 7, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061495919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[2]R generation[0..12]: [9, 9, 7, 6, 8, 8, 9, 2, 4, 9, 9, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061506349Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061517538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061527207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061536907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061546386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061555946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"":"downstairs","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061565765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061574604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"0","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061584314Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"1","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061594073Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"client":"2","":"downstairs","session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061603912Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061614932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"228dcb8c-2b2a-4d5c-9c50-02f720f1d448 is now active with session: 20d9b015-756e-46d7-ac5d-a39ce79cd940","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061625211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.06163406Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"session_id":"20d9b015-756e-46d7-ac5d-a39ce79cd940"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.06164326Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Activated sub_volume 228dcb8c-2b2a-4d5c-9c50-02f720f1d448","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061651399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-18T03:28:42.061659579Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.061910802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"task":"crutest"} 01/30 Write block 250 len 24576 data: 3 3 3 3 3 2 02/30 Read block 592 len 16384 03/30 Write block 105 len 4096 data: 3 04/30 Read block 279 len 24576 05/30 Write block 5 len 40960 data: 3 3 3 3 2 2 2 2 2 3 06/30 Read block 285 len 8192 07/30 Write block 509 len 32768 data: 2 2 2 3 3 3 3 3 08/30 Read block 145 len 12288 09/30 Read block 435 len 20480 10/30 Read block 392 len 24576 11/30 Read block 99 len 4096 12/30 Read block 171 len 8192 13/30 Write block 399 len 20480 data: 2 2 2 2 3 14/30 Write block 104 len 36864 data: 3 4 2 2 2 2 2 2 2 15/30 Read block 338 len 28672 16/30 Write block 438 len 28672 data: 2 2 2 2 2 2 2 17/30 Write block 146 len 36864 data: 2 3 3 3 3 3 2 2 2 18/30 Write block 288 len 16384 data: 2 2 2 3 19/30 Write block 202 len 28672 data: 4 4 5 6 5 4 4 20/30 Read block 97 len 8192 21/30 Read block 467 len 8192 22/30 Read block 498 len 20480 23/30 Write block 410 len 24576 data: 5 5 5 4 2 2 24/30 Read block 468 len 32768 25/30 Read block 567 len 32768 26/30 Read block 167 len 28672 27/30 Write block 282 len 4096 data: 2 28/30 Read block 398 len 20480 29/30 Write block 397 len 12288 data: 2 2 3 30/30 Write block 547 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 24 2 28 Done 28 6 28 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:42.069867239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:42.070051767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 14 14 14 T T T 1 020-039 9 9 9 14 14 14 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 6 6 6 9 9 9 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 8 8 8 13 13 13 T T T 6 120-139 9 9 9 14 14 14 F F F 7 140-159 2 2 2 5 5 5 T F T <--- 8 160-179 4 4 4 7 7 7 F F F 9 180-199 9 9 9 14 14 14 F F F 10 200-219 9 9 9 14 14 14 T F T <--- 11 220-239 8 8 8 13 13 13 F F F 12 240-259 7 7 7 11 11 11 T T T 13 260-279 6 6 6 9 9 9 F F F 14 280-299 6 6 6 9 9 9 T F T <--- 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 9 9 9 14 14 14 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 4 4 4 7 7 7 T F T <--- 20 400-419 9 9 9 14 14 14 T F T <--- 21 420-439 3 3 3 6 6 6 T F T <--- 22 440-459 7 7 7 11 11 11 T F T <--- 23 460-479 7 7 7 11 11 11 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 8 8 8 12 12 12 T F T <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 8 8 8 13 13 13 T F T <--- 28 560-579 8 8 8 13 13 13 F F F 29 580-599 9 9 9 14 14 14 F F F Max gen: 9, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 15 15 15 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 6 6 6 9 9 9 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 9 9 9 14 14 14 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 4 4 4 7 7 7 F F F 9 180-199 9 9 9 14 14 14 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 8 8 8 13 13 13 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 10 10 10 15 15 15 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 9 9 9 14 14 14 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 15 15 15 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 8 8 8 13 13 13 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 10 10 10 15 15 15 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 9 9 9 14 14 14 F F F Max gen: 10, Max flush: 15 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.324778161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333179268Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333486038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333511276Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333519846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333531845Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Crucible 34901f6b-c377-4e7b-b03c-3cc2ec85a7c2 has session id: 4cf05960-baf6-4385-8d67-ad11b803888f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333874463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Upstairs opts: Upstairs UUID: 34901f6b-c377-4e7b-b03c-3cc2ec85a7c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.333904211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Crucible stats registered with UUID: 34901f6b-c377-4e7b-b03c-3cc2ec85a7c2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.33391817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:46.333927769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334053801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.33407077Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334081229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.3342129Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334228529Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"34901f6b-c377-4e7b-b03c-3cc2ec85a7c2 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334249938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334259097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"0","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334269267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"1","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334278986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"2","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334288575Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334298575Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334308624Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"ds_connection connected from Ok(127.0.0.1:65071)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334387849Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42899)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334399768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51601)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.334412437Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335104562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"0","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335141899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"1","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335157158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"2","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.33543132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.33544103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[0]R flush_numbers[0..12]: [15, 14, 11, 9, 13, 15, 14, 15, 7, 14, 15, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335475217Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[0]R generation[0..12]: [10, 9, 7, 6, 8, 10, 9, 10, 4, 9, 10, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335486177Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335497356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[1]R flush_numbers[0..12]: [15, 14, 11, 9, 13, 15, 14, 15, 7, 14, 15, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335507875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[1]R generation[0..12]: [10, 9, 7, 6, 8, 10, 9, 10, 4, 9, 10, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335518194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335529254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[2]R flush_numbers[0..12]: [15, 14, 11, 9, 13, 15, 14, 15, 7, 14, 15, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335539483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[2]R generation[0..12]: [10, 9, 7, 6, 8, 10, 9, 10, 4, 9, 10, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335549612Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335562792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Max found gen is 11","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335572321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Generation requested: 12 >= found:11","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.33558199Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.33559132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335600589Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"":"downstairs","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335610338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335619028Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"0","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335628577Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"1","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335638227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"client":"2","":"downstairs","session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335648106Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335657325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"34901f6b-c377-4e7b-b03c-3cc2ec85a7c2 is now active with session: 4cf05960-baf6-4385-8d67-ad11b803888f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335667405Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335676234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"session_id":"4cf05960-baf6-4385-8d67-ad11b803888f"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335685343Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Activated sub_volume 34901f6b-c377-4e7b-b03c-3cc2ec85a7c2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335693683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335701942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.335994133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"task":"crutest"} 01/30 Write block 324 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Write block 190 len 4096 data: 3 03/30 Write block 166 len 24576 data: 4 4 3 4 4 4 04/30 Flush 05/30 Write block 298 len 12288 data: 2 2 2 06/30 Read block 512 len 24576 07/30 Read block 375 len 16384 08/30 Write block 65 len 4096 data: 3 09/30 Flush 10/30 Write block 290 len 32768 data: 3 4 3 3 2 2 2 2 11/30 Read block 241 len 40960 12/30 Read block 295 len 40960 13/30 Read block 70 len 32768 14/30 Read block 345 len 4096 15/30 Write block 54 len 40960 data: 3 3 4 4 3 3 3 3 3 3 16/30 Write block 211 len 32768 data: 5 6 6 5 4 4 4 3 17/30 Read block 133 len 20480 18/30 Read block 28 len 20480 19/30 Write block 209 len 20480 data: 4 5 6 7 7 20/30 Write block 328 len 24576 data: 3 3 3 3 3 3 21/30 Write block 555 len 16384 data: 3 4 4 4 22/30 Write block 490 len 24576 data: 2 2 2 3 3 2 23/30 Read block 458 len 16384 24/30 Read block 97 len 12288 25/30 Flush 26/30 Write block 50 len 12288 data: 3 3 3 27/30 Read block 57 len 16384 28/30 Write block 487 len 28672 data: 2 2 2 3 3 3 4 29/30 Read block 523 len 8192 30/30 Read block 91 len 8192 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:46.344608957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:46.344775056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 15 15 15 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 12 7 12 18 11 18 T F T <--- 3 060-079 12 6 12 18 9 18 F F F <--- 4 080-099 8 8 8 13 13 13 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 9 9 9 14 14 14 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 12 4 12 16 7 16 F F F <--- 9 180-199 12 9 12 16 14 16 F T F <--- 10 200-219 12 10 12 18 15 18 F F F <--- 11 220-239 8 8 8 13 13 13 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 12 10 12 18 15 18 F F F <--- 15 300-319 12 4 12 17 7 17 F F F <--- 16 320-339 12 1 12 18 1 18 F T F <--- 17 340-359 9 9 9 14 14 14 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 15 15 15 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 12 8 12 18 13 18 T F T <--- 25 500-519 10 10 10 15 15 15 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 12 10 12 18 15 18 F F F <--- 28 560-579 8 8 8 13 13 13 F F F 29 580-599 9 9 9 14 14 14 F F F Max gen: 12, Max flush: 18 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 15 15 15 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 13 13 13 19 19 19 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 9 9 9 14 14 14 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 12 12 12 16 16 16 F F F 9 180-199 12 12 12 16 16 16 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 8 8 8 13 13 13 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 17 17 17 F F F 16 320-339 12 12 12 18 18 18 F F F 17 340-359 9 9 9 14 14 14 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 15 15 15 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 13 13 13 19 19 19 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 12 12 12 18 18 18 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 9 9 9 14 14 14 F F F Max gen: 13, Max flush: 19 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.583187724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.591373955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.591649727Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.591675745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.591684155Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.591696034Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Crucible 7b782b71-039f-4f50-b9ee-02fe650a469c has session id: a5b86ab2-92e7-4639-be94-6e8be6544a3b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592030732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Upstairs opts: Upstairs UUID: 7b782b71-039f-4f50-b9ee-02fe650a469c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.59206288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Crucible stats registered with UUID: 7b782b71-039f-4f50-b9ee-02fe650a469c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592074259Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:50.592083729Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.5922085Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592224579Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592234779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.59236471Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592383719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"7b782b71-039f-4f50-b9ee-02fe650a469c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592411377Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592420716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"0","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592430546Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"1","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592440485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"2","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592450095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592460154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592470093Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48015)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.5925202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56965)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592544928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56475)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.592556647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593165167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"0","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593190066Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"2","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593251642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"1","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593463848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593485136Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[0]R flush_numbers[0..12]: [15, 14, 19, 18, 13, 15, 14, 15, 16, 16, 18, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593566461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[0]R generation[0..12]: [10, 9, 13, 12, 8, 10, 9, 10, 12, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.59357771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.59358909Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[1]R flush_numbers[0..12]: [15, 14, 19, 18, 13, 15, 14, 15, 16, 16, 18, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593599749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[1]R generation[0..12]: [10, 9, 13, 12, 8, 10, 9, 10, 12, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593610308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593621487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[2]R flush_numbers[0..12]: [15, 14, 19, 18, 13, 15, 14, 15, 16, 16, 18, 13]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593631957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[2]R generation[0..12]: [10, 9, 13, 12, 8, 10, 9, 10, 12, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593642446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593653535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593662965Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593672664Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593682073Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593691503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"":"downstairs","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593701462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593710272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"0","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593721821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"1","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.59373149Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"client":"2","":"downstairs","session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.59374155Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593750719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"7b782b71-039f-4f50-b9ee-02fe650a469c is now active with session: a5b86ab2-92e7-4639-be94-6e8be6544a3b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593760788Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593769738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"session_id":"a5b86ab2-92e7-4639-be94-6e8be6544a3b"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593778887Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Activated sub_volume 7b782b71-039f-4f50-b9ee-02fe650a469c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593787077Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.593795156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.594060289Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"task":"crutest"} 01/30 Write block 131 len 4096 data: 4 02/30 Write block 51 len 12288 data: 4 4 3 03/30 Read block 101 len 32768 04/30 Flush 05/30 Write block 179 len 32768 data: 3 3 4 4 3 4 3 3 06/30 Read block 197 len 16384 07/30 Write block 432 len 4096 data: 3 08/30 Write block 451 len 20480 data: 2 2 2 2 2 09/30 Read block 393 len 28672 10/30 Read block 532 len 20480 11/30 Read block 64 len 36864 12/30 Write block 224 len 16384 data: 2 2 2 2 13/30 Flush 14/30 Write block 272 len 12288 data: 2 2 3 15/30 Read block 281 len 12288 16/30 Write block 156 len 24576 data: 2 2 2 3 3 3 17/30 Read block 27 len 28672 18/30 Flush 19/30 Write block 117 len 36864 data: 2 2 2 2 2 2 2 3 3 20/30 Read block 237 len 4096 21/30 Read block 343 len 28672 22/30 Read block 173 len 4096 23/30 Flush 24/30 Write block 577 len 40960 data: 2 2 2 2 2 2 3 4 4 4 25/30 Read block 329 len 20480 26/30 Read block 433 len 40960 27/30 Write block 558 len 16384 data: 5 4 4 4 28/30 Write block 338 len 40960 data: 2 2 2 2 2 2 2 2 2 3 29/30 Write block 492 len 32768 data: 4 5 4 3 2 2 2 2 30/30 Write block 356 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 30 3 36 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:50.602638704Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:50.602794344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1189,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 15 15 15 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 14 13 14 20 19 20 F F F <--- 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 14 10 14 23 15 23 F F F <--- 6 120-139 14 9 14 23 14 23 F F F <--- 7 140-159 14 10 14 22 15 22 F F F <--- 8 160-179 14 12 14 22 16 22 F F F <--- 9 180-199 14 12 14 21 16 21 F F F <--- 10 200-219 12 12 12 18 18 18 F F F 11 220-239 14 8 14 21 13 21 F F F <--- 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 6 14 22 9 22 F F F <--- 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 17 17 17 F F F 16 320-339 12 12 12 18 18 18 T F T <--- 17 340-359 9 9 9 14 14 14 T F T <--- 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 15 15 15 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 14 10 14 21 15 21 F F F <--- 22 440-459 14 10 14 21 15 21 F F F <--- 23 460-479 7 7 7 11 11 11 F F F 24 480-499 13 13 13 19 19 19 T F T <--- 25 500-519 10 10 10 15 15 15 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 12 12 12 18 18 18 T F T <--- 28 560-579 8 8 8 13 13 13 T F T <--- 29 580-599 9 9 9 14 14 14 T F T <--- Max gen: 14, Max flush: 23 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 15 15 15 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 14 14 14 23 23 23 F F F 6 120-139 14 14 14 23 23 23 F F F 7 140-159 14 14 14 22 22 22 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 14 14 14 21 21 21 F F F 10 200-219 12 12 12 18 18 18 F F F 11 220-239 14 14 14 21 21 21 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 12 12 12 18 18 18 F F F 15 300-319 12 12 12 17 17 17 F F F 16 320-339 15 15 15 24 24 24 F F F 17 340-359 15 15 15 24 24 24 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 10 10 10 15 15 15 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 14 14 14 21 21 21 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 15 15 15 24 24 24 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 15 15 15 24 24 24 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 15, Max flush: 24 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.854363626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86267375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.862955821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.862981929Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.862990599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863002318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Crucible 81f6b543-9e91-44fe-8b97-d6272bc4771f has session id: 06a966d1-2484-4d6e-8f67-a193d310932b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863362714Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Upstairs opts: Upstairs UUID: 81f6b543-9e91-44fe-8b97-d6272bc4771f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863384393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Crucible stats registered with UUID: 81f6b543-9e91-44fe-8b97-d6272bc4771f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863394462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:54.863403632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863554092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863569371Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86357946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863722321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86373761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"81f6b543-9e91-44fe-8b97-d6272bc4771f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863761438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863770687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"0","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863780367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"1","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863789906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"2","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863799626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863809675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863821284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50667)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86388789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36354)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863913058Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49711)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.863924827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.864607322Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"1","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.864655049Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"2","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.864666609Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"0","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.864915372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86494991Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[0]R flush_numbers[0..12]: [15, 14, 20, 18, 13, 23, 23, 22, 22, 21, 18, 21]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865017085Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[0]R generation[0..12]: [10, 9, 14, 12, 8, 14, 14, 14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865029585Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865040944Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[1]R flush_numbers[0..12]: [15, 14, 20, 18, 13, 23, 23, 22, 22, 21, 18, 21]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865051723Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[1]R generation[0..12]: [10, 9, 14, 12, 8, 14, 14, 14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865062142Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865073492Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[2]R flush_numbers[0..12]: [15, 14, 20, 18, 13, 23, 23, 22, 22, 21, 18, 21]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865086951Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[2]R generation[0..12]: [10, 9, 14, 12, 8, 14, 14, 14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86509747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865108639Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865118129Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865127928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865137358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865146717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"":"downstairs","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865156606Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865165366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"0","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865175195Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"1","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865184934Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"client":"2","":"downstairs","session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865194794Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865204023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"81f6b543-9e91-44fe-8b97-d6272bc4771f is now active with session: 06a966d1-2484-4d6e-8f67-a193d310932b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865214122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865222892Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"session_id":"06a966d1-2484-4d6e-8f67-a193d310932b"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865233611Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Activated sub_volume 81f6b543-9e91-44fe-8b97-d6272bc4771f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865241701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.86524991Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.865513443Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"task":"crutest"} 01/30 Write block 386 len 40960 data: 2 2 2 2 2 3 3 3 3 3 02/30 Read block 103 len 40960 03/30 Write block 547 len 12288 data: 3 3 3 04/30 Write block 1 len 32768 data: 2 2 2 2 4 4 4 4 05/30 Write block 117 len 32768 data: 3 3 3 3 3 3 3 4 06/30 Flush 07/30 Write block 294 len 12288 data: 3 3 3 08/30 Read block 330 len 20480 09/30 Write block 344 len 12288 data: 3 3 3 10/30 Read block 368 len 40960 11/30 Write block 385 len 40960 data: 2 3 3 3 3 3 4 4 4 4 12/30 Read block 166 len 4096 13/30 Read block 116 len 40960 14/30 Write block 151 len 8192 data: 4 3 15/30 Read block 535 len 40960 16/30 Flush 17/30 Flush 18/30 Write block 350 len 40960 data: 2 2 2 2 2 3 4 4 3 4 19/30 Read block 362 len 32768 20/30 Read block 12 len 28672 21/30 Read block 326 len 16384 22/30 Write block 523 len 4096 data: 2 23/30 Read block 296 len 4096 24/30 Read block 35 len 8192 25/30 Write block 414 len 12288 data: 3 3 2 26/30 Read block 82 len 28672 27/30 Write block 178 len 32768 data: 3 4 4 5 5 4 5 4 28/30 Write block 203 len 40960 data: 5 6 7 6 5 5 5 6 7 8 29/30 Read block 33 len 32768 30/30 Write block 483 len 24576 data: 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 29 29 5 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:54.874927054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:54.875101772Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1197,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 10 25 25 15 F F T <--- 1 020-039 9 9 9 14 14 14 F F F 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 16 16 14 25 25 23 F F T <--- 6 120-139 16 16 14 25 25 23 F F T <--- 7 140-159 16 16 14 26 26 22 F F F <--- 8 160-179 14 14 14 22 22 22 T T F <--- 9 180-199 14 14 14 21 21 21 T T F <--- 10 200-219 12 12 12 18 18 18 T T F <--- 11 220-239 14 14 14 21 21 21 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 16 16 12 26 26 18 F F F <--- 15 300-319 12 12 12 17 17 17 F F F 16 320-339 15 15 15 24 24 24 F F F 17 340-359 16 16 15 26 26 24 T T F <--- 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 10 26 26 15 F F T <--- 20 400-419 10 10 10 15 15 15 T T F <--- 21 420-439 14 14 14 21 21 21 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 15 15 15 24 24 24 T T F <--- 25 500-519 10 10 10 15 15 15 F F F 26 520-539 1 1 1 1 1 1 T T F <--- 27 540-559 16 16 15 25 25 24 F F T <--- 28 560-579 15 15 15 24 24 24 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 16, Max flush: 26 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 25 25 25 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 16 16 16 25 25 25 F F F 6 120-139 16 16 16 25 25 25 F F F 7 140-159 16 16 16 26 26 26 F F F 8 160-179 17 17 17 27 27 27 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 14 14 14 21 21 21 F F F 12 240-259 10 10 10 15 15 15 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 16 16 16 26 26 26 F F F 15 300-319 12 12 12 17 17 17 F F F 16 320-339 15 15 15 24 24 24 F F F 17 340-359 17 17 17 27 27 27 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 17 17 17 27 27 27 F F F 21 420-439 14 14 14 21 21 21 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 17 17 17 27 27 27 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 16 16 16 25 25 25 F F F 28 560-579 15 15 15 24 24 24 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 17, Max flush: 27 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.130157925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.138487118Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.138784588Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.138812856Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.138821456Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.138833215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Crucible cb06699f-a568-48c7-ae1f-b1c150353fdb has session id: 21e4bc93-85ef-43f0-b04c-58a07012063f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139133485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Upstairs opts: Upstairs UUID: cb06699f-a568-48c7-ae1f-b1c150353fdb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139184962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Crucible stats registered with UUID: cb06699f-a568-48c7-ae1f-b1c150353fdb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139195361Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:59.13920464Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139350841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139381339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139392618Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139528189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139543918Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"cb06699f-a568-48c7-ae1f-b1c150353fdb active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139560837Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139569786Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"0","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139579546Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"1","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139589135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"2","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139598754Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139608594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139618673Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58624)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139712277Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49142)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139737555Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46321)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.139748975Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140417331Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"1","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140461198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"2","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140486686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"0","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140706672Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.1407326Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[0]R flush_numbers[0..12]: [25, 14, 20, 18, 13, 25, 25, 26, 27, 27, 27, 21]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140766938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[0]R generation[0..12]: [16, 9, 14, 12, 8, 16, 16, 16, 17, 17, 17, 14]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140778207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140789576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[1]R flush_numbers[0..12]: [25, 14, 20, 18, 13, 25, 25, 26, 27, 27, 27, 21]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140800175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[1]R generation[0..12]: [16, 9, 14, 12, 8, 16, 16, 16, 17, 17, 17, 14]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140810785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140821884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[2]R flush_numbers[0..12]: [25, 14, 20, 18, 13, 25, 25, 26, 27, 27, 27, 21]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140832423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[2]R generation[0..12]: [16, 9, 14, 12, 8, 16, 16, 16, 17, 17, 17, 14]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140842773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140853802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140863191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140872941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.14088388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140893659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"":"downstairs","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140903769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140912768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"0","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140922787Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"1","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140932607Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"client":"2","":"downstairs","session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140942796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140952045Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"cb06699f-a568-48c7-ae1f-b1c150353fdb is now active with session: 21e4bc93-85ef-43f0-b04c-58a07012063f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140962015Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140970854Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"session_id":"21e4bc93-85ef-43f0-b04c-58a07012063f"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140980054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Activated sub_volume cb06699f-a568-48c7-ae1f-b1c150353fdb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140987993Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.140996113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.141298373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"task":"crutest"} 01/30 Write block 453 len 16384 data: 3 3 3 3 02/30 Flush 03/30 Write block 141 len 12288 data: 2 2 2 04/30 Read block 421 len 8192 05/30 Write block 256 len 28672 data: 2 2 2 2 2 2 2 06/30 Write block 299 len 16384 data: 3 3 2 3 07/30 Write block 556 len 28672 data: 5 5 6 5 5 5 4 08/30 Read block 50 len 40960 09/30 Flush 10/30 Write block 546 len 24576 data: 2 4 4 4 2 2 11/30 Write block 470 len 28672 data: 3 3 3 3 2 2 2 12/30 Read block 82 len 12288 13/30 Read block 443 len 32768 14/30 Read block 500 len 40960 15/30 Flush 16/30 Write block 557 len 20480 data: 6 7 6 6 6 17/30 Read block 490 len 12288 18/30 Write block 413 len 40960 data: 5 4 4 3 2 2 2 2 2 3 19/30 Write block 234 len 24576 data: 2 2 2 2 2 3 20/30 Read block 360 len 4096 21/30 Read block 577 len 32768 22/30 Write block 415 len 28672 data: 5 4 3 3 3 3 3 23/30 Read block 213 len 36864 24/30 Read block 479 len 12288 25/30 Read block 521 len 20480 26/30 Read block 184 len 32768 27/30 Write block 430 len 36864 data: 3 3 4 2 2 2 2 2 3 28/30 Write block 553 len 8192 data: 2 2 29/30 Write block 144 len 28672 data: 2 2 3 4 4 4 4 30/30 Read block 131 len 36864 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 2 30 Done 28 0 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1001)) Some(JobId(1014)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:28:59.150031538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:28:59.150198097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 25 25 25 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 16 16 16 25 25 25 F F F 6 120-139 16 16 16 25 25 25 F F F 7 140-159 18 16 18 29 26 29 T F T <--- 8 160-179 17 17 17 27 27 27 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 14 14 14 21 21 21 T F T <--- 12 240-259 18 10 18 29 15 29 F F F <--- 13 260-279 18 14 18 29 22 29 F F F <--- 14 280-299 18 16 18 29 26 29 F F F <--- 15 300-319 18 12 18 29 17 29 F F F <--- 16 320-339 15 15 15 24 24 24 F F F 17 340-359 17 17 17 27 27 27 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 17 17 17 27 27 27 T F T <--- 21 420-439 14 14 14 21 21 21 T F T <--- 22 440-459 18 18 18 28 28 28 F F F 23 460-479 18 7 18 30 11 30 F F F <--- 24 480-499 17 17 17 27 27 27 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 18 16 18 30 25 30 T F T <--- 28 560-579 18 15 18 29 24 29 T F T <--- 29 580-599 15 15 15 24 24 24 F F F Max gen: 18, Max flush: 30 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 25 25 25 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 16 16 16 25 25 25 F F F 6 120-139 16 16 16 25 25 25 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 17 17 17 27 27 27 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 19 19 19 31 31 31 F F F 12 240-259 18 18 18 29 29 29 F F F 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 18 18 18 29 29 29 F F F 16 320-339 15 15 15 24 24 24 F F F 17 340-359 17 17 17 27 27 27 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 19 19 19 31 31 31 F F F 21 420-439 19 19 19 31 31 31 F F F 22 440-459 18 18 18 28 28 28 F F F 23 460-479 18 18 18 30 30 30 F F F 24 480-499 17 17 17 27 27 27 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 15 15 15 24 24 24 F F F Max gen: 19, Max flush: 31 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.393381751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.401656536Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.401951937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.401978205Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.401989554Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402001364Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Crucible f659f188-9a66-4f54-a3ca-6adfb2e2fc1d has session id: 3af6a422-6fe1-44ff-bac2-c367a51b986c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.40236407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Upstairs opts: Upstairs UUID: f659f188-9a66-4f54-a3ca-6adfb2e2fc1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402392778Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Crucible stats registered with UUID: f659f188-9a66-4f54-a3ca-6adfb2e2fc1d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402404187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:03.402413567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402549298Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402564527Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402574546Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402732956Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402748395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"f659f188-9a66-4f54-a3ca-6adfb2e2fc1d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402779763Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402789172Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"0","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402799121Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"1","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.40281124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"2","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.40282089Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402830809Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402840819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61246)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402894135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41218)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402905054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41181)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.402915294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403562161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"1","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.40358228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"0","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403613738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"2","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.40388777Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403899449Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[0]R flush_numbers[0..12]: [25, 14, 20, 18, 13, 25, 25, 31, 27, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403928207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[0]R generation[0..12]: [16, 9, 14, 12, 8, 16, 16, 19, 17, 17, 17, 19]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403939396Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403950626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[1]R flush_numbers[0..12]: [25, 14, 20, 18, 13, 25, 25, 31, 27, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403964035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[1]R generation[0..12]: [16, 9, 14, 12, 8, 16, 16, 19, 17, 17, 17, 19]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403974544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403985483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[2]R flush_numbers[0..12]: [25, 14, 20, 18, 13, 25, 25, 31, 27, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.403995853Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[2]R generation[0..12]: [16, 9, 14, 12, 8, 16, 16, 19, 17, 17, 17, 19]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404017791Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.40402889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.4040385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404049009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404058608Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404068078Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"":"downstairs","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404078057Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404086877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"0","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404096946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"1","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404106905Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"client":"2","":"downstairs","session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404116945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404127724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"f659f188-9a66-4f54-a3ca-6adfb2e2fc1d is now active with session: 3af6a422-6fe1-44ff-bac2-c367a51b986c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404137963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404146803Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"session_id":"3af6a422-6fe1-44ff-bac2-c367a51b986c"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404156062Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Activated sub_volume f659f188-9a66-4f54-a3ca-6adfb2e2fc1d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404164081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404172241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.404464062Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"task":"crutest"} 01/30 Write block 583 len 16384 data: 4 5 5 5 02/30 Write block 20 len 32768 data: 3 3 3 3 3 2 2 2 03/30 Read block 415 len 40960 04/30 Read block 482 len 16384 05/30 Read block 460 len 20480 06/30 Read block 357 len 4096 07/30 Flush 08/30 Write block 445 len 28672 data: 2 2 2 2 2 2 3 09/30 Read block 379 len 16384 10/30 Read block 463 len 4096 11/30 Write block 90 len 28672 data: 2 2 2 2 2 2 2 12/30 Read block 397 len 16384 13/30 Read block 10 len 32768 14/30 Write block 138 len 20480 data: 2 2 2 3 3 15/30 Read block 359 len 24576 16/30 Read block 119 len 12288 17/30 Read block 259 len 20480 18/30 Write block 242 len 36864 data: 3 2 2 2 2 2 2 3 4 19/30 Read block 49 len 28672 20/30 Write block 318 len 36864 data: 2 2 2 2 2 2 3 3 3 21/30 Read block 328 len 12288 22/30 Read block 392 len 36864 23/30 Read block 235 len 32768 24/30 Flush 25/30 Write block 135 len 4096 data: 2 26/30 Write block 204 len 32768 data: 7 8 7 6 6 6 7 8 27/30 Read block 523 len 8192 28/30 Write block 340 len 16384 data: 3 3 3 3 29/30 Read block 503 len 36864 30/30 Write block 149 len 24576 data: 5 5 5 4 3 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Write 8 Done Done Sent false 1002 Acked Read 10 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:03.413245244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:03.413416613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 25 25 25 F F F 1 020-039 20 20 9 32 32 14 F F F <--- 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 20 20 8 33 33 13 F F F <--- 5 100-119 16 16 16 25 25 25 F F F 6 120-139 20 20 16 33 33 25 T T F <--- 7 140-159 20 20 19 33 33 31 T T F <--- 8 160-179 17 17 17 27 27 27 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 T T F <--- 11 220-239 19 19 19 31 31 31 F F F 12 240-259 20 20 18 33 33 29 F F F <--- 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 20 20 18 33 33 29 F F F <--- 16 320-339 20 20 15 33 33 24 F F F <--- 17 340-359 17 17 17 27 27 27 T T F <--- 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 19 19 19 31 31 31 F F F 21 420-439 19 19 19 31 31 31 F F F 22 440-459 20 20 18 33 33 28 F F F <--- 23 460-479 18 18 18 30 30 30 F F F 24 480-499 17 17 17 27 27 27 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 20 20 15 32 32 24 F F F <--- Max gen: 20, Max flush: 33 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 25 25 25 F F F 1 020-039 20 20 20 32 32 32 F F F 2 040-059 14 14 14 20 20 20 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 20 20 20 33 33 33 F F F 5 100-119 16 16 16 25 25 25 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 21 21 21 34 34 34 F F F 8 160-179 17 17 17 27 27 27 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 19 19 19 31 31 31 F F F 12 240-259 20 20 20 33 33 33 F F F 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 21 21 21 34 34 34 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 19 19 19 31 31 31 F F F 21 420-439 19 19 19 31 31 31 F F F 22 440-459 20 20 20 33 33 33 F F F 23 460-479 18 18 18 30 30 30 F F F 24 480-499 17 17 17 27 27 27 F F F 25 500-519 10 10 10 15 15 15 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 21, Max flush: 34 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.656068332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:07.66340281Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.663465936Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.670805293Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671069856Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671096064Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671104583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671116382Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Crucible 26a5a8f2-31aa-428b-af41-f22b9179e225 has session id: 3449ed8f-2aff-4a98-9dc7-2f79315747fe","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.67145448Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Upstairs opts: Upstairs UUID: 26a5a8f2-31aa-428b-af41-f22b9179e225, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671477479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Crucible stats registered with UUID: 26a5a8f2-31aa-428b-af41-f22b9179e225","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671489868Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:07.671498857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671663986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671692995Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671703914Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671813297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671828626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"26a5a8f2-31aa-428b-af41-f22b9179e225 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671861083Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671870653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"0","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671880552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"1","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671890362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"2","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.671899981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.67190988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.67192009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:39790)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.672013613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:44041)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.672041992Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:43409)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.672155024Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.672707608Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"1","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.672725867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"2","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.672753235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"0","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673056045Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673067384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[0]R flush_numbers[0..12]: [25, 32, 20, 18, 33, 25, 34, 34, 27, 27, 34, 31]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673080593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[0]R generation[0..12]: [16, 20, 14, 12, 20, 16, 21, 21, 17, 17, 21, 19]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673091313Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673102552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[1]R flush_numbers[0..12]: [25, 32, 20, 18, 33, 25, 34, 34, 27, 27, 34, 31]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673113271Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[1]R generation[0..12]: [16, 20, 14, 12, 20, 16, 21, 21, 17, 17, 21, 19]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.6731236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.67313508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[2]R flush_numbers[0..12]: [25, 32, 20, 18, 33, 25, 34, 34, 27, 27, 34, 31]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673145759Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[2]R generation[0..12]: [16, 20, 14, 12, 20, 16, 21, 21, 17, 17, 21, 19]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673158358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673169327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673178867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673188676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673198236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673207605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"":"downstairs","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673217454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673226304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"0","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673236183Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"1","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673245882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"client":"2","":"downstairs","session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673255872Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673265081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"26a5a8f2-31aa-428b-af41-f22b9179e225 is now active with session: 3449ed8f-2aff-4a98-9dc7-2f79315747fe","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.6732752Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.67328398Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"session_id":"3449ed8f-2aff-4a98-9dc7-2f79315747fe"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673293119Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Activated sub_volume 26a5a8f2-31aa-428b-af41-f22b9179e225","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673301169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673309278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1221,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.673576741Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} 01/30 Write block 228 len 20480 data: 2 2 2 2 2 02/30 Read block 582 len 16384 03/30 Read block 461 len 8192 04/30 Write block 28 len 28672 data: 2 2 2 2 2 2 2 05/30 Read block 356 len 40960 06/30 Flush 07/30 Write block 549 len 8192 data: 5 3 08/30 Flush 09/30 Write block 141 len 36864 data: 4 4 3 3 3 4 5 5 6 10/30 Read block 96 len 12288 11/30 Read block 530 len 4096 12/30 Read block 109 len 24576 13/30 Read block 55 len 28672 14/30 Write block 142 len 36864 data: 5 4 4 4 5 6 6 7 6 15/30 Read block 447 len 24576 16/30 Flush 17/30 Flush 18/30 Write block 84 len 36864 data: 2 2 2 2 2 2 3 3 3 19/30 Write block 3 len 40960 data: 3 3 5 5 5 5 3 3 3 3 20/30 Write block 47 len 8192 data: 3 3 21/30 Write block 142 len 8192 data: 6 5 22/30 Write block 171 len 8192 data: 5 4 23/30 Write block 319 len 24576 data: 3 3 3 3 3 4 24/30 Write block 8 len 12288 data: 6 4 4 25/30 Read block 387 len 20480 26/30 Write block 499 len 32768 data: 3 2 2 2 2 2 2 2 27/30 Read block 111 len 24576 28/30 Write block 403 len 4096 data: 4 29/30 Write block 307 len 8192 data: 3 3 30/30 Read block 392 len 4096 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 2 30 Done 2 30 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:07.681845137Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:07.682000597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1221,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 25 25 25 F T T <--- 1 020-039 20 22 22 32 35 35 F F F <--- 2 040-059 14 14 14 20 20 20 F T T <--- 3 060-079 12 12 12 18 18 18 F F F 4 080-099 20 20 20 33 33 33 F T T <--- 5 100-119 16 16 16 25 25 25 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 21 22 22 34 37 37 F T T <--- 8 160-179 17 17 17 27 27 27 F T T <--- 9 180-199 17 17 17 27 27 27 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 19 22 22 31 35 35 T F F <--- 12 240-259 20 20 20 33 33 33 F F F 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 20 20 20 33 33 33 F T T <--- 16 320-339 20 20 20 33 33 33 F T T <--- 17 340-359 21 21 21 34 34 34 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 19 19 19 31 31 31 F T T <--- 21 420-439 19 19 19 31 31 31 F F F 22 440-459 20 20 20 33 33 33 F F F 23 460-479 18 18 18 30 30 30 F F F 24 480-499 17 17 17 27 27 27 F T T <--- 25 500-519 10 10 10 15 15 15 F T T <--- 26 520-539 17 17 17 27 27 27 F F F 27 540-559 19 22 22 31 36 36 F F F <--- 28 560-579 19 19 19 31 31 31 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 22, Max flush: 37 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 22 22 22 35 35 35 F F F 2 040-059 23 23 23 38 38 38 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 16 16 16 25 25 25 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 23 23 23 38 38 38 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 22 22 22 35 35 35 F F F 12 240-259 20 20 20 33 33 33 F F F 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 23 23 23 38 38 38 F F F 16 320-339 23 23 23 38 38 38 F F F 17 340-359 21 21 21 34 34 34 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 23 23 23 38 38 38 F F F 21 420-439 19 19 19 31 31 31 F F F 22 440-459 20 20 20 33 33 33 F F F 23 460-479 18 18 18 30 30 30 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 23 23 23 38 38 38 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 22 22 22 36 36 36 F F F 28 560-579 19 19 19 31 31 31 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 23, Max flush: 38 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.94237988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.950786087Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951075948Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951103696Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951112296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951124085Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Crucible 89f56039-943e-4ecb-b8aa-7ee766922677 has session id: 5897e3fc-4dbc-4dd7-b0f0-19ee224c7066","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951447524Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Upstairs opts: Upstairs UUID: 89f56039-943e-4ecb-b8aa-7ee766922677, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951470912Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Crucible stats registered with UUID: 89f56039-943e-4ecb-b8aa-7ee766922677","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951480711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:11.951489881Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951634721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.95164906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.9516593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.95181005Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951825769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"89f56039-943e-4ecb-b8aa-7ee766922677 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951837968Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951846897Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"0","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951856707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"1","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951866466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"2","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951876135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951886105Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951895884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"ds_connection connected from Ok(127.0.0.1:60211)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.951980469Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"ds_connection connected from Ok(127.0.0.1:63842)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952067923Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"ds_connection connected from Ok(127.0.0.1:35519)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952089521Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952614677Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"0","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952643755Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"1","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952665184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"2","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952917207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.952956214Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[0]R flush_numbers[0..12]: [38, 35, 38, 18, 38, 25, 34, 38, 38, 27, 34, 35]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953014591Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[0]R generation[0..12]: [23, 22, 23, 12, 23, 16, 21, 23, 23, 17, 21, 22]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.95302574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953037249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[1]R flush_numbers[0..12]: [38, 35, 38, 18, 38, 25, 34, 38, 38, 27, 34, 35]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953047898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[1]R generation[0..12]: [23, 22, 23, 12, 23, 16, 21, 23, 23, 17, 21, 22]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953058138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953069157Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[2]R flush_numbers[0..12]: [38, 35, 38, 18, 38, 25, 34, 38, 38, 27, 34, 35]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953079666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[2]R generation[0..12]: [23, 22, 23, 12, 23, 16, 21, 23, 23, 17, 21, 22]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953090166Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953101105Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953110404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953119994Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953129353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953138632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"":"downstairs","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953150002Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953158751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"0","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.9531683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"1","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.95317811Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"client":"2","":"downstairs","session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953188059Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953197269Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"89f56039-943e-4ecb-b8aa-7ee766922677 is now active with session: 5897e3fc-4dbc-4dd7-b0f0-19ee224c7066","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953207278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953215967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"session_id":"5897e3fc-4dbc-4dd7-b0f0-19ee224c7066"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953225017Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Activated sub_volume 89f56039-943e-4ecb-b8aa-7ee766922677","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953239496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953247765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.953539656Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"task":"crutest"} 01/30 Write block 515 len 28672 data: 4 4 4 4 2 2 2 02/30 Write block 210 len 36864 data: 8 9 9 8 6 5 5 5 4 03/30 Write block 233 len 32768 data: 2 3 3 3 3 3 4 3 04/30 Read block 502 len 4096 05/30 Read block 268 len 36864 06/30 Read block 229 len 4096 07/30 Write block 337 len 36864 data: 2 3 3 4 4 4 4 4 4 08/30 Read block 482 len 8192 09/30 Read block 196 len 16384 10/30 Read block 258 len 8192 11/30 Write block 560 len 28672 data: 7 7 5 3 2 2 3 12/30 Write block 452 len 20480 data: 3 4 4 4 4 13/30 Write block 245 len 28672 data: 3 3 3 3 4 5 4 14/30 Write block 445 len 40960 data: 3 3 3 3 3 3 4 4 5 5 15/30 Flush 16/30 Write block 50 len 20480 data: 4 5 5 4 4 17/30 Read block 65 len 20480 18/30 Read block 416 len 20480 19/30 Write block 336 len 36864 data: 2 3 4 4 5 5 5 5 5 20/30 Read block 284 len 28672 21/30 Read block 298 len 40960 22/30 Write block 235 len 40960 data: 4 4 4 4 5 4 3 4 3 3 23/30 Write block 435 len 28672 data: 3 3 3 4 3 3 3 24/30 Write block 103 len 24576 data: 4 4 5 3 3 3 25/30 Read block 267 len 36864 26/30 Read block 344 len 20480 27/30 Write block 465 len 4096 data: 2 28/30 Read block 575 len 12288 29/30 Write block 319 len 8192 data: 4 4 30/30 Write block 499 len 16384 data: 4 3 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 1 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 27 31 Done 28 28 3 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:11.961926944Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:11.962100563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1229,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 22 22 22 35 35 35 F F F 2 040-059 23 23 23 38 38 38 T T F <--- 3 060-079 12 12 12 18 18 18 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 16 16 16 25 25 25 T T F <--- 6 120-139 21 21 21 34 34 34 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 23 23 23 38 38 38 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 24 24 21 39 39 34 F F T <--- 11 220-239 24 24 22 39 39 35 T T T <--- 12 240-259 24 24 20 39 39 33 T T T <--- 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 23 23 23 38 38 38 T T F <--- 16 320-339 24 24 23 39 39 38 T T F <--- 17 340-359 24 24 21 39 39 34 T T F <--- 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 23 23 23 38 38 38 F F F 21 420-439 19 19 19 31 31 31 T T F <--- 22 440-459 24 24 20 39 39 33 T T F <--- 23 460-479 18 18 18 30 30 30 T T F <--- 24 480-499 23 23 23 38 38 38 T T F <--- 25 500-519 24 24 23 39 39 38 T T T <--- 26 520-539 24 24 17 39 39 27 F F T <--- 27 540-559 22 22 22 36 36 36 F F F 28 560-579 24 24 19 39 39 31 F F F <--- 29 580-599 20 20 20 32 32 32 F F F Max gen: 24, Max flush: 39 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 22 22 22 35 35 35 F F F 2 040-059 25 25 25 40 40 40 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 25 25 25 40 40 40 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 23 23 23 38 38 38 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 24 24 24 39 39 39 F F F 11 220-239 25 25 25 40 40 40 F F F 12 240-259 25 25 25 40 40 40 F F F 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 F F F 15 300-319 25 25 25 40 40 40 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 25 25 25 40 40 40 F F F 18 360-379 8 8 8 12 12 12 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 23 23 23 38 38 38 F F F 21 420-439 25 25 25 40 40 40 F F F 22 440-459 25 25 25 40 40 40 F F F 23 460-479 25 25 25 40 40 40 F F F 24 480-499 25 25 25 40 40 40 F F F 25 500-519 25 25 25 40 40 40 F F F 26 520-539 24 24 24 39 39 39 F F F 27 540-559 22 22 22 36 36 36 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 25, Max flush: 40 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.220493726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.228598113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.228888594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.228931661Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.22894333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.228955829Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Crucible 81d1f24c-af09-49d5-a045-980109b6b953 has session id: 7e543e77-df26-4047-834c-cbde68643c3b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229334444Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Upstairs opts: Upstairs UUID: 81d1f24c-af09-49d5-a045-980109b6b953, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229364092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Crucible stats registered with UUID: 81d1f24c-af09-49d5-a045-980109b6b953","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229375162Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:16.229384641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229522222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229539711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.22954993Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.22970006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.22971248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"81d1f24c-af09-49d5-a045-980109b6b953 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229730318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229739248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"0","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229748937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"1","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229758876Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"2","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229770266Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229780255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229790294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61869)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.229865229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62551)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230022759Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53354)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230051147Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230542285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"1","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230568133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"2","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230579243Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"0","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230875773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.23091979Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[0]R flush_numbers[0..12]: [38, 35, 40, 18, 38, 40, 34, 38, 38, 27, 39, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.230988816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[0]R generation[0..12]: [23, 22, 25, 12, 23, 25, 21, 23, 23, 17, 24, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231000035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231011284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[1]R flush_numbers[0..12]: [38, 35, 40, 18, 38, 40, 34, 38, 38, 27, 39, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231021683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[1]R generation[0..12]: [23, 22, 25, 12, 23, 25, 21, 23, 23, 17, 24, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231034533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231045562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[2]R flush_numbers[0..12]: [38, 35, 40, 18, 38, 40, 34, 38, 38, 27, 39, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231056021Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[2]R generation[0..12]: [23, 22, 25, 12, 23, 25, 21, 23, 23, 17, 24, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.23106646Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.23107734Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231086799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231096549Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231105788Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231115157Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"":"downstairs","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231124817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231133446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"0","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231143185Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"1","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231152855Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"client":"2","":"downstairs","session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231162654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231171794Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"81d1f24c-af09-49d5-a045-980109b6b953 is now active with session: 7e543e77-df26-4047-834c-cbde68643c3b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231183223Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231192062Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"session_id":"7e543e77-df26-4047-834c-cbde68643c3b"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231201212Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Activated sub_volume 81d1f24c-af09-49d5-a045-980109b6b953","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231209321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231217501Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.231521901Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"task":"crutest"} 01/30 Write block 123 len 4096 data: 4 02/30 Read block 182 len 12288 03/30 Read block 194 len 16384 04/30 Read block 489 len 20480 05/30 Flush 06/30 Write block 199 len 8192 data: 4 4 07/30 Write block 247 len 28672 data: 4 4 5 6 5 4 4 08/30 Write block 354 len 16384 data: 3 4 5 5 09/30 Write block 244 len 20480 data: 4 4 4 5 5 10/30 Read block 395 len 8192 11/30 Read block 422 len 40960 12/30 Read block 393 len 36864 13/30 Write block 360 len 40960 data: 3 3 3 3 4 3 4 4 4 4 14/30 Read block 205 len 4096 15/30 Write block 36 len 12288 data: 2 2 2 16/30 Read block 242 len 28672 17/30 Read block 116 len 28672 18/30 Flush 19/30 Write block 422 len 40960 data: 4 2 3 3 3 3 3 3 4 4 20/30 Read block 467 len 4096 21/30 Flush 22/30 Flush 23/30 Write block 294 len 40960 data: 4 4 4 3 3 4 4 3 4 3 24/30 Read block 281 len 12288 25/30 Write block 475 len 32768 data: 3 3 2 2 2 2 2 2 26/30 Read block 315 len 16384 27/30 Write block 196 len 36864 data: 3 4 4 5 5 4 5 6 8 28/30 Write block 362 len 28672 data: 4 4 5 4 5 5 5 29/30 Read block 352 len 32768 30/30 Read block 506 len 16384 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 3 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:16.240231288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:16.240406696Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1237,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 22 42 42 35 F F F <--- 2 040-059 25 25 25 40 40 40 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 25 25 25 40 40 40 F F F 6 120-139 26 26 21 41 41 34 F F F <--- 7 140-159 23 23 23 38 38 38 F F F 8 160-179 23 23 23 38 38 38 F F F 9 180-199 26 26 17 42 42 27 T T F <--- 10 200-219 26 26 24 42 42 39 T T F <--- 11 220-239 25 25 25 40 40 40 F F F 12 240-259 26 26 25 42 42 40 F F F <--- 13 260-279 18 18 18 29 29 29 F F F 14 280-299 18 18 18 29 29 29 T T F <--- 15 300-319 25 25 25 40 40 40 T T F <--- 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 25 42 42 40 F F F <--- 18 360-379 26 26 8 42 42 12 T T F <--- 19 380-399 16 16 16 26 26 26 F F F 20 400-419 23 23 23 38 38 38 F F F 21 420-439 26 26 25 43 43 40 F F F <--- 22 440-459 25 25 25 40 40 40 F F F 23 460-479 25 25 25 40 40 40 T T F <--- 24 480-499 25 25 25 40 40 40 T T F <--- 25 500-519 25 25 25 40 40 40 F F F 26 520-539 24 24 24 39 39 39 F F F 27 540-559 22 22 22 36 36 36 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 26, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 25 25 25 40 40 40 F F F 3 060-079 12 12 12 18 18 18 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 25 25 25 40 40 40 F F F 6 120-139 26 26 26 41 41 41 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 23 23 23 38 38 38 F F F 9 180-199 27 27 27 44 44 44 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 25 25 25 40 40 40 F F F 12 240-259 26 26 26 42 42 42 F F F 13 260-279 18 18 18 29 29 29 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 27 27 27 44 44 44 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 27 27 27 44 44 44 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 23 23 23 38 38 38 F F F 21 420-439 26 26 26 43 43 43 F F F 22 440-459 25 25 25 40 40 40 F F F 23 460-479 27 27 27 44 44 44 F F F 24 480-499 27 27 27 44 44 44 F F F 25 500-519 25 25 25 40 40 40 F F F 26 520-539 24 24 24 39 39 39 F F F 27 540-559 22 22 22 36 36 36 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 27, Max flush: 44 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.491726775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.499972483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500252745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500277703Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500286492Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500298152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Crucible 8624581f-0aa6-4039-9936-796d7640edc6 has session id: 652b4caf-19bd-4c2c-8e31-b37820f7a82b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500659468Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Upstairs opts: Upstairs UUID: 8624581f-0aa6-4039-9936-796d7640edc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500689366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Crucible stats registered with UUID: 8624581f-0aa6-4039-9936-796d7640edc6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500700385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:20.500712494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500845596Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500861135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.500871324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501020304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501032133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"8624581f-0aa6-4039-9936-796d7640edc6 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501065561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.50107498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"0","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.50108498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"1","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501094689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"2","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501104488Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501114538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501124507Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64223)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501177424Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57637)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501203072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64379)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501214561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501892257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"2","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501911475Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"0","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.501967122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"1","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502214375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502237604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[0]R flush_numbers[0..12]: [38, 42, 40, 18, 38, 40, 41, 38, 38, 44, 44, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502280551Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[0]R generation[0..12]: [23, 26, 25, 12, 23, 25, 26, 23, 23, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.50229226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.50230378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[1]R flush_numbers[0..12]: [38, 42, 40, 18, 38, 40, 41, 38, 38, 44, 44, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502314299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[1]R generation[0..12]: [23, 26, 25, 12, 23, 25, 26, 23, 23, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502325018Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502336117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[2]R flush_numbers[0..12]: [38, 42, 40, 18, 38, 40, 41, 38, 38, 44, 44, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502346547Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[2]R generation[0..12]: [23, 26, 25, 12, 23, 25, 26, 23, 23, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502357116Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502368305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502380285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502390104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502399503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502409003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"":"downstairs","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502418932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502427671Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"0","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502437451Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"1","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.50244717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"client":"2","":"downstairs","session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.50245699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502466219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"8624581f-0aa6-4039-9936-796d7640edc6 is now active with session: 652b4caf-19bd-4c2c-8e31-b37820f7a82b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502476498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502485308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"session_id":"652b4caf-19bd-4c2c-8e31-b37820f7a82b"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502494567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Activated sub_volume 8624581f-0aa6-4039-9936-796d7640edc6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502502627Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502510916Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.502776049Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"task":"crutest"} 01/30 Flush 02/30 Write block 537 len 8192 data: 2 2 03/30 Read block 90 len 24576 04/30 Write block 359 len 24576 data: 5 4 4 5 5 6 05/30 Read block 593 len 16384 06/30 Read block 128 len 28672 07/30 Write block 272 len 20480 data: 3 3 4 3 2 08/30 Read block 427 len 16384 09/30 Read block 503 len 28672 10/30 Flush 11/30 Write block 362 len 24576 data: 6 6 7 5 6 6 12/30 Write block 472 len 36864 data: 4 4 3 4 4 3 3 3 3 13/30 Write block 574 len 20480 data: 2 2 2 3 3 14/30 Write block 395 len 40960 data: 4 2 3 3 4 3 3 3 5 4 15/30 Write block 442 len 8192 data: 3 3 16/30 Read block 175 len 32768 17/30 Write block 162 len 40960 data: 3 3 3 3 5 5 4 5 5 6 18/30 Read block 231 len 28672 19/30 Flush 20/30 Write block 310 len 32768 data: 3 2 2 2 2 2 2 2 21/30 Write block 69 len 4096 data: 2 22/30 Read block 182 len 8192 23/30 Read block 458 len 36864 24/30 Write block 525 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Read block 540 len 16384 26/30 Read block 491 len 12288 27/30 Write block 513 len 40960 data: 4 4 5 5 5 5 3 3 3 2 28/30 Read block 229 len 16384 29/30 Write block 43 len 16384 data: 2 3 3 3 30/30 Read block 555 len 16384 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 29 1 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1000)) Some(JobId(1018)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:20.512274534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:20.512444963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1245,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 25 25 25 40 40 40 T F T <--- 3 060-079 12 12 12 18 18 18 T F T <--- 4 080-099 23 23 23 38 38 38 F F F 5 100-119 25 25 25 40 40 40 F F F 6 120-139 26 26 26 41 41 41 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 28 23 28 47 38 47 F F F <--- 9 180-199 27 27 27 44 44 44 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 25 25 25 40 40 40 F F F 12 240-259 26 26 26 42 42 42 F F F 13 260-279 28 18 28 46 29 46 F F F <--- 14 280-299 27 27 27 44 44 44 F F F 15 300-319 27 27 27 44 44 44 T F T <--- 16 320-339 25 25 25 40 40 40 F F F 17 340-359 28 26 28 46 42 46 F F F <--- 18 360-379 28 27 28 47 44 47 F F F <--- 19 380-399 28 16 28 47 26 47 F F F <--- 20 400-419 28 23 28 47 38 47 F F F <--- 21 420-439 26 26 26 43 43 43 F F F 22 440-459 28 25 28 47 40 47 F F F <--- 23 460-479 28 27 28 47 44 47 F F F <--- 24 480-499 28 27 28 47 44 47 F F F <--- 25 500-519 25 25 25 40 40 40 T F T <--- 26 520-539 28 24 28 46 39 46 T T T <--- 27 540-559 22 22 22 36 36 36 F F F 28 560-579 28 24 28 47 39 47 F F F <--- 29 580-599 20 20 20 32 32 32 F F F Max gen: 28, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 25 25 25 40 40 40 F F F 6 120-139 26 26 26 41 41 41 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 28 28 28 47 47 47 F F F 9 180-199 27 27 27 44 44 44 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 25 25 25 40 40 40 F F F 12 240-259 26 26 26 42 42 42 F F F 13 260-279 28 28 28 46 46 46 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 29 29 29 48 48 48 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 28 28 28 47 47 47 F F F 21 420-439 26 26 26 43 43 43 F F F 22 440-459 28 28 28 47 47 47 F F F 23 460-479 28 28 28 47 47 47 F F F 24 480-499 28 28 28 47 47 47 F F F 25 500-519 29 29 29 48 48 48 F F F 26 520-539 29 29 29 48 48 48 F F F 27 540-559 22 22 22 36 36 36 F F F 28 560-579 28 28 28 47 47 47 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 29, Max flush: 48 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.765249514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.773424117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.773703708Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.773733766Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.773742226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.773754045Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Crucible d0579944-3ac6-4a66-8f23-befe3b95d99d has session id: 43df865a-2049-4c4b-a41b-f715d67badca","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774016618Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Upstairs opts: Upstairs UUID: d0579944-3ac6-4a66-8f23-befe3b95d99d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774061395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Crucible stats registered with UUID: d0579944-3ac6-4a66-8f23-befe3b95d99d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774071894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:24.774080803Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774222514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774254572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774265541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774388663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774401222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"d0579944-3ac6-4a66-8f23-befe3b95d99d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.7744357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774448059Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"0","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774458089Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"1","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774468068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"2","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774477807Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774487667Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774497656Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"ds_connection connected from Ok(127.0.0.1:42336)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774564072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"ds_connection connected from Ok(127.0.0.1:61240)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.77458945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"ds_connection connected from Ok(127.0.0.1:64956)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.774655726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775268415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"2","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775290574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"1","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775319062Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"0","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775574735Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775584594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[0]R flush_numbers[0..12]: [38, 42, 48, 48, 38, 40, 41, 38, 47, 44, 44, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775598814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[0]R generation[0..12]: [23, 26, 29, 29, 23, 25, 26, 23, 28, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775611683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775623032Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[1]R flush_numbers[0..12]: [38, 42, 48, 48, 38, 40, 41, 38, 47, 44, 44, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775633611Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[1]R generation[0..12]: [23, 26, 29, 29, 23, 25, 26, 23, 28, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775643941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.77565503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[2]R flush_numbers[0..12]: [38, 42, 48, 48, 38, 40, 41, 38, 47, 44, 44, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775665709Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[2]R generation[0..12]: [23, 26, 29, 29, 23, 25, 26, 23, 28, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775676168Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775687018Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775696307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775705877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775715036Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775724515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"":"downstairs","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775734415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775743264Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"0","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775752973Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"1","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775764003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"client":"2","":"downstairs","session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775773792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775782941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"d0579944-3ac6-4a66-8f23-befe3b95d99d is now active with session: 43df865a-2049-4c4b-a41b-f715d67badca","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775792891Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.77580165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"session_id":"43df865a-2049-4c4b-a41b-f715d67badca"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.77581099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Activated sub_volume d0579944-3ac6-4a66-8f23-befe3b95d99d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775819079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.775827229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.776120509Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"task":"crutest"} 01/30 Write block 540 len 24576 data: 2 2 2 2 2 2 02/30 Flush 03/30 Write block 550 len 28672 data: 4 3 2 3 3 4 6 04/30 Flush 05/30 Write block 107 len 40960 data: 4 4 3 3 3 3 2 2 2 2 06/30 Flush 07/30 Write block 227 len 16384 data: 3 3 3 3 08/30 Read block 589 len 36864 09/30 Read block 559 len 28672 10/30 Read block 553 len 24576 11/30 Write block 575 len 12288 data: 3 3 4 12/30 Read block 577 len 12288 13/30 Write block 494 len 24576 data: 5 4 3 3 3 5 14/30 Read block 37 len 20480 15/30 Write block 481 len 36864 data: 3 3 3 3 3 3 4 4 3 16/30 Flush 17/30 Write block 73 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Write block 545 len 32768 data: 3 3 5 5 6 5 4 3 19/30 Write block 402 len 40960 data: 4 6 5 4 4 4 4 5 6 6 20/30 Read block 476 len 28672 21/30 Read block 216 len 12288 22/30 Write block 197 len 12288 data: 5 5 6 23/30 Read block 333 len 40960 24/30 Read block 540 len 36864 25/30 Write block 407 len 8192 data: 5 5 26/30 Write block 425 len 12288 data: 4 4 4 27/30 Read block 213 len 32768 28/30 Read block 312 len 12288 29/30 Read block 558 len 24576 30/30 Write block 268 len 36864 data: 2 2 2 2 4 4 5 4 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 25 0 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1003)) Some(JobId(1015)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:24.785223481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:24.78538835Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1253,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 29 29 29 48 48 48 T F T <--- 4 080-099 23 23 23 38 38 38 T F T <--- 5 100-119 30 25 30 51 40 51 F F F <--- 6 120-139 26 26 26 41 41 41 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 28 28 28 47 47 47 F F F 9 180-199 27 27 27 44 44 44 T F T <--- 10 200-219 27 27 27 44 44 44 F F F 11 220-239 30 25 30 52 40 52 F F F <--- 12 240-259 26 26 26 42 42 42 F F F 13 260-279 28 28 28 46 46 46 T F T <--- 14 280-299 27 27 27 44 44 44 F F F 15 300-319 29 29 29 48 48 48 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 28 28 28 47 47 47 T F T <--- 21 420-439 26 26 26 43 43 43 T F T <--- 22 440-459 28 28 28 47 47 47 F F F 23 460-479 28 28 28 47 47 47 F F F 24 480-499 30 28 30 52 47 52 F F F <--- 25 500-519 29 29 29 48 48 48 F F F 26 520-539 29 29 29 48 48 48 F F F 27 540-559 30 30 30 50 50 50 T F T <--- 28 560-579 30 28 30 52 47 52 F F F <--- 29 580-599 20 20 20 32 32 32 F F F Max gen: 30, Max flush: 52 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 30 30 30 51 51 51 F F F 6 120-139 26 26 26 41 41 41 F F F 7 140-159 23 23 23 38 38 38 F F F 8 160-179 28 28 28 47 47 47 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 26 26 26 42 42 42 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 29 29 29 48 48 48 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 28 28 28 47 47 47 F F F 23 460-479 28 28 28 47 47 47 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 29 29 29 48 48 48 F F F 26 520-539 29 29 29 48 48 48 F F F 27 540-559 31 31 31 53 53 53 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 31, Max flush: 53 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.030351526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:29.037616098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.037670525Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.04473673Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.044990463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045016812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045025801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.04503765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Crucible b13ab424-fc34-4e1b-a992-c93b0fdddf68 has session id: 1691c8e1-d9ad-475b-83a6-290b872f1f36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045403466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Upstairs opts: Upstairs UUID: b13ab424-fc34-4e1b-a992-c93b0fdddf68, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045427345Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Crucible stats registered with UUID: b13ab424-fc34-4e1b-a992-c93b0fdddf68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045437854Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:29.045446963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045590544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045619772Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045630641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045776842Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.04580083Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"b13ab424-fc34-4e1b-a992-c93b0fdddf68 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045818449Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045827498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"0","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045837188Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"1","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045846867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"2","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045856456Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045866676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045876685Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"ds_connection connected from Ok(127.0.0.1:63278)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045919812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"ds_connection connected from Ok(127.0.0.1:41228)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045930681Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"ds_connection connected from Ok(127.0.0.1:58184)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.045940991Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046570559Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"2","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046612877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"1","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046656934Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"0","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.04687098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046898068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[0]R flush_numbers[0..12]: [38, 42, 48, 53, 53, 51, 41, 38, 47, 53, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046972213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[0]R generation[0..12]: [23, 26, 29, 31, 31, 30, 26, 23, 28, 31, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046983382Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.046994651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[1]R flush_numbers[0..12]: [38, 42, 48, 53, 53, 51, 41, 38, 47, 53, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047005421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[1]R generation[0..12]: [23, 26, 29, 31, 31, 30, 26, 23, 28, 31, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.04701589Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047027009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[2]R flush_numbers[0..12]: [38, 42, 48, 53, 53, 51, 41, 38, 47, 53, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047039699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[2]R generation[0..12]: [23, 26, 29, 31, 31, 30, 26, 23, 28, 31, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047050288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047061227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047070616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047080296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047089685Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047099135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"":"downstairs","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047109334Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047118273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"0","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047128283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"1","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047138272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"client":"2","":"downstairs","session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047148481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047157831Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"b13ab424-fc34-4e1b-a992-c93b0fdddf68 is now active with session: 1691c8e1-d9ad-475b-83a6-290b872f1f36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.0471679Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047177189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"session_id":"1691c8e1-d9ad-475b-83a6-290b872f1f36"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047188309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Activated sub_volume b13ab424-fc34-4e1b-a992-c93b0fdddf68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047196378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047204488Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.047507958Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} 01/30 Write block 152 len 32768 data: 5 4 4 2 3 3 3 4 02/30 Flush 03/30 Write block 598 len 4096 data: 3 04/30 Read block 223 len 32768 05/30 Flush 06/30 Write block 101 len 12288 data: 3 4 5 07/30 Write block 387 len 28672 data: 4 4 4 4 5 5 5 08/30 Read block 433 len 20480 09/30 Write block 512 len 36864 data: 4 5 5 6 6 6 6 4 4 10/30 Write block 173 len 20480 data: 4 4 3 2 3 11/30 Write block 463 len 4096 data: 2 12/30 Read block 219 len 4096 13/30 Write block 496 len 20480 data: 4 4 4 6 4 14/30 Write block 550 len 28672 data: 6 5 4 4 4 5 7 15/30 Read block 482 len 16384 16/30 Read block 302 len 40960 17/30 Read block 552 len 36864 18/30 Write block 449 len 8192 data: 4 4 19/30 Read block 294 len 8192 20/30 Write block 422 len 24576 data: 5 3 4 5 5 5 21/30 Write block 485 len 24576 data: 4 4 5 5 4 4 22/30 Write block 459 len 8192 data: 3 3 23/30 Read block 495 len 20480 24/30 Write block 515 len 8192 data: 7 7 25/30 Read block 11 len 20480 26/30 Write block 283 len 16384 data: 2 2 3 3 27/30 Read block 533 len 40960 28/30 Read block 162 len 36864 29/30 Read block 512 len 40960 30/30 Write block 547 len 24576 data: 6 6 7 7 6 5 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 0 24 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:29.05598441Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:29.056155889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1261,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 30 30 30 51 51 51 F T T <--- 6 120-139 26 26 26 41 41 41 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 28 28 28 47 47 47 F T T <--- 9 180-199 31 31 31 53 53 53 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 26 26 26 42 42 42 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 27 27 27 44 44 44 F T T <--- 15 300-319 29 29 29 48 48 48 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 28 28 28 47 47 47 F T T <--- 20 400-419 31 31 31 53 53 53 F F F 21 420-439 31 31 31 53 53 53 F T T <--- 22 440-459 28 28 28 47 47 47 F T T <--- 23 460-479 28 28 28 47 47 47 F T T <--- 24 480-499 30 30 30 52 52 52 F T T <--- 25 500-519 29 29 29 48 48 48 F T T <--- 26 520-539 29 29 29 48 48 48 F T T <--- 27 540-559 31 31 31 53 53 53 F T T <--- 28 560-579 30 30 30 52 52 52 F F F 29 580-599 32 32 32 55 55 55 F F F Max gen: 32, Max flush: 55 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 31 31 31 53 53 53 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 33 33 33 56 56 56 F F F 6 120-139 26 26 26 41 41 41 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 56 56 56 F F F 9 180-199 31 31 31 53 53 53 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 30 30 30 52 52 52 F F F 12 240-259 26 26 26 42 42 42 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 33 33 33 56 56 56 F F F 15 300-319 29 29 29 48 48 48 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 33 33 33 56 56 56 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 33 33 33 56 56 56 F F F 24 480-499 33 33 33 56 56 56 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 33 33 33 56 56 56 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 32 32 32 55 55 55 F F F Max gen: 33, Max flush: 56 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.313281337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.32160558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.321885801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.321911739Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.321920209Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.321931868Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Crucible 3d6814a6-7fb8-4061-b36f-549c8e780c78 has session id: 57ed5e3b-a4d4-48c1-80c2-973b76a6d61f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322221659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Upstairs opts: Upstairs UUID: 3d6814a6-7fb8-4061-b36f-549c8e780c78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322250867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Crucible stats registered with UUID: 3d6814a6-7fb8-4061-b36f-549c8e780c78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322261916Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:33.322271296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322396698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322412217Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322424606Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322583165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322598174Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"3d6814a6-7fb8-4061-b36f-549c8e780c78 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322614683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322623513Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"0","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322633282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"1","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322642801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"2","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322652311Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.3226625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322672709Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40572)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.32281863Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48790)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322843108Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50304)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.322854927Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323475307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"2","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323539752Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"1","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323606218Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"0","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323812484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323827483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[0]R flush_numbers[0..12]: [38, 42, 48, 53, 53, 56, 41, 54, 56, 53, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.32388071Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[0]R generation[0..12]: [23, 26, 29, 31, 31, 33, 26, 32, 33, 31, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323891959Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323903118Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[1]R flush_numbers[0..12]: [38, 42, 48, 53, 53, 56, 41, 54, 56, 53, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323913678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[1]R generation[0..12]: [23, 26, 29, 31, 31, 33, 26, 32, 33, 31, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323924107Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323935076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[2]R flush_numbers[0..12]: [38, 42, 48, 53, 53, 56, 41, 54, 56, 53, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323945616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[2]R generation[0..12]: [23, 26, 29, 31, 31, 33, 26, 32, 33, 31, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323955945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323967044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323976514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323985983Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.323995342Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324006542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"":"downstairs","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324016321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.32402515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"0","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.3240349Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"1","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324044699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"client":"2","":"downstairs","session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324054678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324063798Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"3d6814a6-7fb8-4061-b36f-549c8e780c78 is now active with session: 57ed5e3b-a4d4-48c1-80c2-973b76a6d61f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324073797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324082477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"session_id":"57ed5e3b-a4d4-48c1-80c2-973b76a6d61f"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324091636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Activated sub_volume 3d6814a6-7fb8-4061-b36f-549c8e780c78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324099716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324107895Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.324358299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"task":"crutest"} 01/30 Write block 72 len 32768 data: 2 3 3 3 3 3 3 3 02/30 Write block 281 len 20480 data: 2 3 3 3 4 03/30 Read block 73 len 4096 04/30 Write block 98 len 16384 data: 3 3 3 4 05/30 Write block 321 len 40960 data: 4 4 4 5 4 4 3 4 4 4 06/30 Read block 51 len 36864 07/30 Flush 08/30 Write block 284 len 20480 data: 4 5 4 2 3 09/30 Write block 237 len 28672 data: 5 5 6 5 4 5 4 10/30 Read block 291 len 36864 11/30 Write block 0 len 16384 data: 2 3 3 4 12/30 Read block 363 len 4096 13/30 Read block 29 len 8192 14/30 Write block 186 len 16384 data: 4 4 4 3 15/30 Read block 273 len 16384 16/30 Read block 468 len 40960 17/30 Read block 112 len 16384 18/30 Write block 209 len 24576 data: 7 9 10 10 9 7 19/30 Write block 73 len 20480 data: 4 4 4 4 4 20/30 Read block 310 len 24576 21/30 Write block 392 len 32768 data: 6 6 5 5 3 4 4 5 22/30 Write block 218 len 12288 data: 5 3 3 23/30 Write block 119 len 24576 data: 4 4 4 4 5 5 24/30 Flush 25/30 Write block 540 len 32768 data: 3 3 3 3 3 4 4 7 26/30 Flush 27/30 Write block 590 len 36864 data: 4 4 4 3 3 3 3 3 4 28/30 Read block 537 len 16384 29/30 Read block 306 len 32768 30/30 Read block 114 len 24576 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:33.3334593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:33.333621949Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1269,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 23 34 58 38 58 F F F <--- 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 34 31 34 58 53 58 F T F <--- 4 080-099 34 31 34 57 53 57 F F F <--- 5 100-119 34 33 34 58 56 58 F F F <--- 6 120-139 34 26 34 58 41 58 F F F <--- 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 56 56 56 F F F 9 180-199 34 31 34 58 53 58 F F F <--- 10 200-219 34 27 34 58 44 58 F F F <--- 11 220-239 34 30 34 58 52 58 F F F <--- 12 240-259 34 26 34 58 42 58 F F F <--- 13 260-279 31 31 31 53 53 53 F F F 14 280-299 34 33 34 58 56 58 F T F <--- 15 300-319 29 29 29 48 48 48 F F F 16 320-339 34 25 34 57 40 57 F F F <--- 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 34 33 34 58 56 58 F F F <--- 20 400-419 31 31 31 53 53 53 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 33 33 33 56 56 56 F F F 24 480-499 33 33 33 56 56 56 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 33 34 59 56 59 F F F <--- 28 560-579 30 30 30 52 52 52 F F F 29 580-599 32 32 32 55 55 55 T F T <--- Max gen: 34, Max flush: 59 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 F F F 1 020-039 26 26 26 42 42 42 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 34 34 34 58 58 58 F F F 4 080-099 34 34 34 57 57 57 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 56 56 56 F F F 9 180-199 34 34 34 58 58 58 F F F 10 200-219 34 34 34 58 58 58 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 29 29 29 48 48 48 F F F 16 320-339 34 34 34 57 57 57 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 33 33 33 56 56 56 F F F 24 480-499 33 33 33 56 56 56 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 35 35 35 60 60 60 F F F Max gen: 35, Max flush: 60 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.583521232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:37.590791964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.59085244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598130841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598396034Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598422822Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598431261Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598442951Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Crucible 68b635d9-e694-4536-9536-8be0d969a510 has session id: d43f12dc-2861-4902-ae08-cc863a4a1f30","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598727402Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Upstairs opts: Upstairs UUID: 68b635d9-e694-4536-9536-8be0d969a510, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.59874852Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Crucible stats registered with UUID: 68b635d9-e694-4536-9536-8be0d969a510","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.59875862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:37.598772589Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598933528Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598948287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.598958297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599079879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599098018Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"68b635d9-e694-4536-9536-8be0d969a510 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599130165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599139595Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"0","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599151044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"1","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599160773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"2","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599170283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599180032Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599189781Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"ds_connection connected from Ok(127.0.0.1:44149)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599276276Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"ds_connection connected from Ok(127.0.0.1:58560)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599397928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"ds_connection connected from Ok(127.0.0.1:58956)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599422736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599861587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"2","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599938562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"0","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.599949891Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"1","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600133019Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600213354Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[0]R flush_numbers[0..12]: [58, 42, 48, 58, 57, 58, 58, 54, 56, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600266311Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[0]R generation[0..12]: [34, 26, 29, 34, 34, 34, 34, 32, 33, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.60027736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600291169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[1]R flush_numbers[0..12]: [58, 42, 48, 58, 57, 58, 58, 54, 56, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600302048Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[1]R generation[0..12]: [34, 26, 29, 34, 34, 34, 34, 32, 33, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600312548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600323517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[2]R flush_numbers[0..12]: [58, 42, 48, 58, 57, 58, 58, 54, 56, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600333936Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[2]R generation[0..12]: [34, 26, 29, 34, 34, 34, 34, 32, 33, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600344226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600355215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600364684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600374294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600383503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600392982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"":"downstairs","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600402662Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600411491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"0","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.60042107Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"1","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.60043068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"client":"2","":"downstairs","session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600442039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600451298Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"68b635d9-e694-4536-9536-8be0d969a510 is now active with session: d43f12dc-2861-4902-ae08-cc863a4a1f30","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600461408Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600470417Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600477947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"session_id":"d43f12dc-2861-4902-ae08-cc863a4a1f30"} {"msg":"Activated sub_volume 68b635d9-e694-4536-9536-8be0d969a510","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600487206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600495206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.600768018Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} 01/30 Write block 298 len 16384 data: 4 5 5 4 02/30 Write block 469 len 16384 data: 3 4 4 5 03/30 Write block 594 len 24576 data: 4 4 4 4 5 3 04/30 Write block 206 len 16384 data: 8 7 7 8 05/30 Write block 300 len 8192 data: 6 5 06/30 Write block 478 len 20480 data: 4 4 4 4 4 07/30 Read block 262 len 36864 08/30 Read block 223 len 40960 09/30 Read block 197 len 12288 10/30 Write block 302 len 20480 data: 5 4 4 3 3 11/30 Read block 309 len 16384 12/30 Read block 189 len 8192 13/30 Read block 558 len 36864 14/30 Flush 15/30 Write block 17 len 40960 data: 3 3 3 4 4 4 4 4 3 3 16/30 Read block 363 len 20480 17/30 Write block 10 len 12288 data: 5 4 4 18/30 Read block 254 len 16384 19/30 Read block 341 len 24576 20/30 Write block 147 len 32768 data: 7 7 8 7 6 6 5 5 21/30 Read block 472 len 24576 22/30 Write block 471 len 8192 data: 5 6 23/30 Read block 434 len 28672 24/30 Read block 388 len 36864 25/30 Read block 532 len 28672 26/30 Read block 505 len 4096 27/30 Read block 171 len 40960 28/30 Read block 514 len 40960 29/30 Write block 125 len 12288 data: 4 4 4 30/30 Write block 69 len 16384 data: 3 2 2 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 4 28 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:37.608590943Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:37.608769551Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 F T T <--- 1 020-039 26 26 26 42 42 42 F T T <--- 2 040-059 29 29 29 48 48 48 F F F 3 060-079 34 34 34 58 58 58 F T T <--- 4 080-099 34 34 34 57 57 57 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F T T <--- 7 140-159 32 32 32 54 54 54 F T T <--- 8 160-179 33 33 33 56 56 56 F F F 9 180-199 34 34 34 58 58 58 F F F 10 200-219 34 36 36 58 61 61 T F F <--- 11 220-239 34 34 34 58 58 58 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 34 36 36 58 61 61 T F F <--- 15 300-319 29 36 36 48 61 61 T F F <--- 16 320-339 34 34 34 57 57 57 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 33 36 36 56 61 61 T T T <--- 24 480-499 33 36 36 56 61 61 F F F <--- 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 35 36 36 60 61 61 T F F <--- Max gen: 36, Max flush: 61 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 37 37 37 62 62 62 F F F 4 080-099 34 34 34 57 57 57 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 33 33 33 56 56 56 F F F 9 180-199 34 34 34 58 58 58 F F F 10 200-219 36 36 36 61 61 61 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 36 36 36 61 61 61 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 34 34 34 57 57 57 F F F 17 340-359 28 28 28 46 46 46 F F F 18 360-379 28 28 28 47 47 47 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 37 37 37 62 62 62 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 36 36 36 61 61 61 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.860374061Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.868831754Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869122385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869149064Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869157823Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869169622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Crucible 4cb3e7ef-a666-4ba5-9260-9efc396df9da has session id: 4bf3a922-818b-48c8-ada4-8e5401c6baf1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869499971Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Upstairs opts: Upstairs UUID: 4cb3e7ef-a666-4ba5-9260-9efc396df9da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869520339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Crucible stats registered with UUID: 4cb3e7ef-a666-4ba5-9260-9efc396df9da","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869532748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:41.869541768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869684498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869713926Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869725076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869858167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869877736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"4cb3e7ef-a666-4ba5-9260-9efc396df9da active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869906324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869915913Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"0","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869925583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"1","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869935292Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"2","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869944941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.869954941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.86996498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47980)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.870081052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57407)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.87011776Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40711)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.870144268Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.870708821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"0","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.87072095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"2","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.870758328Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"1","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871043239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871052768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[0]R flush_numbers[0..12]: [62, 62, 48, 62, 57, 58, 62, 62, 56, 58, 61, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871119504Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[0]R generation[0..12]: [37, 37, 29, 37, 34, 34, 37, 37, 33, 34, 36, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871130633Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871141723Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[1]R flush_numbers[0..12]: [62, 62, 48, 62, 57, 58, 62, 62, 56, 58, 61, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871152342Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[1]R generation[0..12]: [37, 37, 29, 37, 34, 34, 37, 37, 33, 34, 36, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871162641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.8711736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[2]R flush_numbers[0..12]: [62, 62, 48, 62, 57, 58, 62, 62, 56, 58, 61, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.87118429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[2]R generation[0..12]: [37, 37, 29, 37, 34, 34, 37, 37, 33, 34, 36, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871194749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871207778Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871217328Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871226967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871236286Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871245566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"":"downstairs","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871255345Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871264135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"0","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871273774Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"1","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871283293Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"client":"2","":"downstairs","session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871293243Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871302452Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"4cb3e7ef-a666-4ba5-9260-9efc396df9da is now active with session: 4bf3a922-818b-48c8-ada4-8e5401c6baf1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871312561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871321281Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"session_id":"4bf3a922-818b-48c8-ada4-8e5401c6baf1"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.87133038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Activated sub_volume 4cb3e7ef-a666-4ba5-9260-9efc396df9da","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.87133851Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871346569Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.871622861Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"task":"crutest"} 01/30 Write block 132 len 4096 data: 2 02/30 Read block 113 len 4096 03/30 Write block 266 len 12288 data: 2 2 3 04/30 Write block 596 len 16384 data: 5 5 6 4 05/30 Write block 294 len 16384 data: 5 5 5 4 06/30 Read block 497 len 36864 07/30 Read block 528 len 36864 08/30 Write block 357 len 40960 data: 6 4 6 5 5 7 7 8 6 7 09/30 Write block 195 len 20480 data: 3 4 6 6 7 10/30 Read block 540 len 16384 11/30 Write block 193 len 28672 data: 2 3 4 5 7 7 8 12/30 Read block 546 len 4096 13/30 Read block 248 len 8192 14/30 Read block 411 len 32768 15/30 Write block 211 len 32768 data: 11 11 10 8 6 6 6 6 16/30 Write block 399 len 16384 data: 6 4 4 5 17/30 Read block 126 len 20480 18/30 Read block 374 len 40960 19/30 Write block 151 len 24576 data: 7 7 6 6 3 4 20/30 Read block 544 len 24576 21/30 Write block 198 len 24576 data: 8 9 6 5 6 7 22/30 Read block 386 len 36864 23/30 Flush 24/30 Write block 402 len 20480 data: 6 7 6 5 5 25/30 Write block 69 len 32768 data: 4 3 3 4 5 5 5 5 26/30 Write block 144 len 36864 data: 5 5 6 8 8 9 8 8 8 27/30 Read block 487 len 24576 28/30 Write block 368 len 32768 data: 6 5 4 3 3 3 2 2 29/30 Read block 335 len 8192 30/30 Read block 504 len 40960 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 19 0 19 Done 30 11 30 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:41.880574052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:41.880733672Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 37 37 37 62 62 62 T F T <--- 4 080-099 34 34 34 57 57 57 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 38 37 38 63 62 63 F T F <--- 7 140-159 38 37 38 63 62 63 T F T <--- 8 160-179 33 33 33 56 56 56 F F F 9 180-199 38 34 38 63 58 63 F T F <--- 10 200-219 38 36 38 63 61 63 F F F <--- 11 220-239 34 34 34 58 58 58 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 38 31 38 63 53 63 F T F <--- 14 280-299 38 36 38 63 61 63 F T F <--- 15 300-319 36 36 36 61 61 61 F F F 16 320-339 34 34 34 57 57 57 F F F 17 340-359 38 28 38 63 46 63 F T F <--- 18 360-379 38 28 38 63 47 63 T T T <--- 19 380-399 38 34 38 63 58 63 F F F <--- 20 400-419 38 31 38 63 53 63 T F T <--- 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 37 37 37 62 62 62 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 38 36 38 63 61 63 F T F <--- Max gen: 38, Max flush: 63 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 34 34 34 57 57 57 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 38 38 38 63 63 63 F F F 7 140-159 39 39 39 64 64 64 F F F 8 160-179 33 33 33 56 56 56 F F F 9 180-199 38 38 38 63 63 63 F F F 10 200-219 38 38 38 63 63 63 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 34 34 34 58 58 58 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 38 38 38 63 63 63 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 34 34 34 57 57 57 F F F 17 340-359 38 38 38 63 63 63 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 38 38 38 63 63 63 F F F 20 400-419 39 39 39 64 64 64 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 33 33 33 56 56 56 F F F 23 460-479 37 37 37 62 62 62 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 38 38 38 63 63 63 F F F Max gen: 39, Max flush: 64 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.128332406Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.136637399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.136947039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.136985816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.136995376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137007355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Crucible 064eeabc-e394-4620-b599-9189f2080491 has session id: 1638aaa8-3208-4959-9159-89fafd81f5c5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137325284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Upstairs opts: Upstairs UUID: 064eeabc-e394-4620-b599-9189f2080491, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137354532Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Crucible stats registered with UUID: 064eeabc-e394-4620-b599-9189f2080491","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137365801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:46.137375271Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137506132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137521771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137532001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.13768588Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137704969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"064eeabc-e394-4620-b599-9189f2080491 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137733287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137742817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"0","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137752556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"1","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137762385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"2","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137771945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137781964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137791843Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:56293)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137862169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46471)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.137887407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53300)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.13799748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138568732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"0","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138581882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"1","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138681775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"2","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138900511Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138930969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[0]R flush_numbers[0..12]: [62, 62, 48, 64, 57, 58, 63, 64, 56, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138982475Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[0]R generation[0..12]: [37, 37, 29, 39, 34, 34, 38, 39, 33, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.138993684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139004824Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[1]R flush_numbers[0..12]: [62, 62, 48, 64, 57, 58, 63, 64, 56, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139015533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[1]R generation[0..12]: [37, 37, 29, 39, 34, 34, 38, 39, 33, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139026112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139037082Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[2]R flush_numbers[0..12]: [62, 62, 48, 64, 57, 58, 63, 64, 56, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139047691Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[2]R generation[0..12]: [37, 37, 29, 39, 34, 34, 38, 39, 33, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.1390583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139069749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139079339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139089128Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139098478Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139107867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"":"downstairs","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139119216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139128026Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"0","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139137675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"1","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139147234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"client":"2","":"downstairs","session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139157054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139166223Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"064eeabc-e394-4620-b599-9189f2080491 is now active with session: 1638aaa8-3208-4959-9159-89fafd81f5c5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139176282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139185092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"session_id":"1638aaa8-3208-4959-9159-89fafd81f5c5"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139194201Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Activated sub_volume 064eeabc-e394-4620-b599-9189f2080491","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139202231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.13921044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.139491712Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"task":"crutest"} 01/30 Write block 318 len 24576 data: 3 5 5 5 5 5 02/30 Write block 570 len 8192 data: 3 3 03/30 Write block 446 len 24576 data: 4 4 4 5 5 5 04/30 Write block 157 len 28672 data: 4 4 5 4 4 4 4 05/30 Read block 355 len 24576 06/30 Write block 388 len 4096 data: 5 07/30 Read block 435 len 12288 08/30 Read block 2 len 36864 09/30 Read block 7 len 8192 10/30 Read block 97 len 36864 11/30 Flush 12/30 Write block 567 len 28672 data: 3 3 3 4 4 3 2 13/30 Write block 97 len 36864 data: 2 4 4 4 5 5 6 5 6 14/30 Flush 15/30 Write block 346 len 32768 data: 4 4 2 2 3 3 3 3 16/30 Read block 536 len 8192 17/30 Read block 428 len 40960 18/30 Write block 286 len 20480 data: 5 3 4 3 4 19/30 Write block 335 len 36864 data: 2 3 4 5 5 6 6 6 6 20/30 Read block 562 len 32768 21/30 Write block 462 len 24576 data: 2 3 2 3 2 3 22/30 Write block 314 len 24576 data: 3 3 3 3 4 6 23/30 Read block 92 len 32768 24/30 Write block 242 len 24576 data: 6 5 5 5 5 6 25/30 Read block 33 len 28672 26/30 Read block 273 len 16384 27/30 Write block 116 len 28672 data: 3 4 4 5 5 5 5 28/30 Write block 0 len 36864 data: 3 4 4 5 4 6 6 6 7 29/30 Write block 173 len 4096 data: 5 30/30 Write block 227 len 36864 data: 4 4 4 4 3 3 3 4 5 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 6 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 28 34 Done 27 27 2 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:46.148005762Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:46.148158252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1293,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 T T F <--- 1 020-039 37 37 37 62 62 62 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 40 40 34 66 66 57 F F F <--- 5 100-119 40 40 34 66 66 58 T T F <--- 6 120-139 38 38 38 63 63 63 T T F <--- 7 140-159 40 40 39 65 65 64 F F F <--- 8 160-179 40 40 33 65 65 56 T T F <--- 9 180-199 38 38 38 63 63 63 F F F 10 200-219 38 38 38 63 63 63 F F F 11 220-239 34 34 34 58 58 58 T T F <--- 12 240-259 34 34 34 58 58 58 T T F <--- 13 260-279 38 38 38 63 63 63 F F F 14 280-299 38 38 38 63 63 63 T T F <--- 15 300-319 40 40 36 65 65 61 T T T <--- 16 320-339 40 40 34 65 65 57 T T T <--- 17 340-359 38 38 38 63 63 63 T T F <--- 18 360-379 39 39 39 64 64 64 F F F 19 380-399 40 40 38 65 65 63 F F F <--- 20 400-419 39 39 39 64 64 64 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 40 40 33 65 65 56 F F F <--- 23 460-479 37 37 37 62 62 62 T T F <--- 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 40 40 30 66 66 52 F F T <--- 29 580-599 38 38 38 63 63 63 F F F Max gen: 40, Max flush: 66 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 67 67 67 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 29 29 29 48 48 48 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 41 41 41 67 67 67 F F F 6 120-139 41 41 41 67 67 67 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 41 41 41 67 67 67 F F F 9 180-199 38 38 38 63 63 63 F F F 10 200-219 38 38 38 63 63 63 F F F 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 41 41 41 67 67 67 F F F 15 300-319 41 41 41 67 67 67 F F F 16 320-339 41 41 41 67 67 67 F F F 17 340-359 41 41 41 67 67 67 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 40 40 40 65 65 65 F F F 20 400-419 39 39 39 64 64 64 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 40 40 40 65 65 65 F F F 23 460-479 41 41 41 67 67 67 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 40 40 40 66 66 66 F F F 29 580-599 38 38 38 63 63 63 F F F Max gen: 41, Max flush: 67 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.409116036Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:50.416333591Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.416388318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.423449503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.423714536Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.423740554Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.423751623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.423763442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Crucible e193dd24-71a6-4c9f-a6a9-10e90e2109ea has session id: cbe620b9-1477-4c09-beeb-ba6c22767545","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424072472Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Upstairs opts: Upstairs UUID: e193dd24-71a6-4c9f-a6a9-10e90e2109ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.42410463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Crucible stats registered with UUID: e193dd24-71a6-4c9f-a6a9-10e90e2109ea","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424116019Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:50.424125409Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424274679Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424290088Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424300147Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424447567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424462986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"e193dd24-71a6-4c9f-a6a9-10e90e2109ea active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424499304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424508723Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"0","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424518383Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"1","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424530312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"2","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424540161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424550051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.42455997Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"ds_connection connected from Ok(127.0.0.1:36284)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424635405Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"ds_connection connected from Ok(127.0.0.1:47442)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424660833Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"ds_connection connected from Ok(127.0.0.1:61908)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.424672173Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425348278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"2","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425380386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"1","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425410994Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"0","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425662388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425684896Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[0]R flush_numbers[0..12]: [67, 62, 48, 64, 66, 67, 67, 65, 67, 63, 63, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425762321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[0]R generation[0..12]: [41, 37, 29, 39, 40, 41, 41, 40, 41, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.42577402Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425785499Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[1]R flush_numbers[0..12]: [67, 62, 48, 64, 66, 67, 67, 65, 67, 63, 63, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425799099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[1]R generation[0..12]: [41, 37, 29, 39, 40, 41, 41, 40, 41, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425809738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425820817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[2]R flush_numbers[0..12]: [67, 62, 48, 64, 66, 67, 67, 65, 67, 63, 63, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425831216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[2]R generation[0..12]: [41, 37, 29, 39, 40, 41, 41, 40, 41, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425841566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425852565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425862034Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425871744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425881043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425890473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"":"downstairs","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425900162Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425909141Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"0","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425918981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"1","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.42592883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"client":"2","":"downstairs","session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425940429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425949639Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"e193dd24-71a6-4c9f-a6a9-10e90e2109ea is now active with session: cbe620b9-1477-4c09-beeb-ba6c22767545","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425959688Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425968487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"session_id":"cbe620b9-1477-4c09-beeb-ba6c22767545"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425977687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Activated sub_volume e193dd24-71a6-4c9f-a6a9-10e90e2109ea","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425985806Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.425993936Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.426327404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} 01/30 Write block 141 len 12288 data: 5 7 6 02/30 Read block 191 len 40960 03/30 Flush 04/30 Write block 47 len 20480 data: 4 4 3 5 6 05/30 Read block 534 len 24576 06/30 Read block 582 len 40960 07/30 Flush 08/30 Write block 142 len 12288 data: 8 7 6 09/30 Read block 398 len 32768 10/30 Read block 290 len 28672 11/30 Read block 134 len 28672 12/30 Write block 108 len 24576 data: 5 4 4 4 4 3 13/30 Read block 191 len 20480 14/30 Write block 181 len 16384 data: 6 6 5 6 15/30 Read block 403 len 36864 16/30 Write block 285 len 28672 data: 6 6 4 5 4 5 5 17/30 Flush 18/30 Flush 19/30 Write block 555 len 32768 data: 6 8 7 8 7 8 8 6 20/30 Read block 310 len 12288 21/30 Read block 132 len 12288 22/30 Read block 570 len 28672 23/30 Read block 575 len 40960 24/30 Read block 214 len 40960 25/30 Read block 589 len 12288 26/30 Write block 208 len 32768 data: 8 9 10 12 12 11 9 7 27/30 Write block 390 len 40960 data: 5 6 7 7 6 6 4 5 5 7 28/30 Write block 327 len 12288 data: 4 5 5 29/30 Read block 328 len 32768 30/30 Write block 304 len 32768 data: 5 4 4 4 4 3 4 3 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Done Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 1 1 24 Done 1 22 22 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:50.436149118Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:50.436298048Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1301,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 67 67 67 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 41 42 42 67 70 70 F F F <--- 6 120-139 41 41 41 67 67 67 F F F 7 140-159 42 42 42 68 70 70 T F F <--- 8 160-179 41 41 41 67 67 67 F F F 9 180-199 38 42 42 63 70 70 F F F <--- 10 200-219 38 38 38 63 63 63 F T T <--- 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 41 42 42 67 70 70 F F F <--- 15 300-319 41 41 41 67 67 67 F T T <--- 16 320-339 41 41 41 67 67 67 F T T <--- 17 340-359 41 41 41 67 67 67 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 40 40 40 65 65 65 F T T <--- 20 400-419 39 39 39 64 64 64 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 40 40 40 65 65 65 F F F 23 460-479 41 41 41 67 67 67 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 59 59 59 F T T <--- 28 560-579 40 40 40 66 66 66 F T T <--- 29 580-599 38 38 38 63 63 63 F F F Max gen: 42, Max flush: 70 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 67 67 67 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 41 41 41 67 67 67 F F F 7 140-159 42 42 42 70 70 70 F F F 8 160-179 41 41 41 67 67 67 F F F 9 180-199 42 42 42 70 70 70 F F F 10 200-219 43 43 43 71 71 71 F F F 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 43 43 43 71 71 71 F F F 16 320-339 43 43 43 71 71 71 F F F 17 340-359 41 41 41 67 67 67 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 39 39 39 64 64 64 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 40 40 40 65 65 65 F F F 23 460-479 41 41 41 67 67 67 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 43 43 43 71 71 71 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 38 38 38 63 63 63 F F F Max gen: 43, Max flush: 71 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.687401511Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.695775651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696059272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696096559Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696105929Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696118208Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Crucible 6797acef-eb18-40f7-8b74-183fae971034 has session id: 7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696426148Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Upstairs opts: Upstairs UUID: 6797acef-eb18-40f7-8b74-183fae971034, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696479524Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Crucible stats registered with UUID: 6797acef-eb18-40f7-8b74-183fae971034","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696490064Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:54.696501213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696633454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696662562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696673542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696790104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696805153Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"6797acef-eb18-40f7-8b74-183fae971034 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696823242Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696832131Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"0","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.69684183Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"1","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.69685163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"2","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696861109Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696871129Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696881208Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"ds_connection connected from Ok(127.0.0.1:32820)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.696967452Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65489)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.69699614Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34299)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.69700782Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.697664046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"2","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.697692775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"1","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.697788558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"0","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698001644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698016903Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[0]R flush_numbers[0..12]: [67, 62, 69, 64, 66, 70, 67, 70, 67, 70, 71, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698082099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[0]R generation[0..12]: [41, 37, 42, 39, 40, 42, 41, 42, 41, 42, 43, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698093318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698104597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[1]R flush_numbers[0..12]: [67, 62, 69, 64, 66, 70, 67, 70, 67, 70, 71, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698115107Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[1]R generation[0..12]: [41, 37, 42, 39, 40, 42, 41, 42, 41, 42, 43, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698125566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698136515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[2]R flush_numbers[0..12]: [67, 62, 69, 64, 66, 70, 67, 70, 67, 70, 71, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698147165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[2]R generation[0..12]: [41, 37, 42, 39, 40, 42, 41, 42, 41, 42, 43, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698157714Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698168723Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698180862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698190462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698199771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698209251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"":"downstairs","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698228119Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"0","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698237919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"1","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698247578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"client":"2","":"downstairs","session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698257557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698266877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"6797acef-eb18-40f7-8b74-183fae971034 is now active with session: 7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698276926Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698285616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"session_id":"7eaf9ec0-6ef6-4606-ae3c-6eae8089dfe4"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698294835Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Activated sub_volume 6797acef-eb18-40f7-8b74-183fae971034","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698302724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698310784Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.698587986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"task":"crutest"} 01/30 Write block 133 len 8192 data: 2 2 02/30 Read block 154 len 28672 03/30 Read block 43 len 4096 04/30 Read block 39 len 40960 05/30 Write block 559 len 24576 data: 8 9 9 7 4 3 06/30 Read block 19 len 12288 07/30 Flush 08/30 Write block 345 len 40960 data: 5 5 5 3 3 4 4 4 4 4 09/30 Read block 304 len 40960 10/30 Flush 11/30 Write block 405 len 8192 data: 6 6 12/30 Flush 13/30 Write block 120 len 24576 data: 6 6 6 6 6 5 14/30 Read block 358 len 24576 15/30 Read block 520 len 24576 16/30 Read block 33 len 32768 17/30 Flush 18/30 Write block 132 len 24576 data: 3 3 3 3 2 2 19/30 Read block 341 len 40960 20/30 Write block 203 len 28672 data: 8 9 9 9 8 9 10 21/30 Read block 132 len 8192 22/30 Flush 23/30 Write block 409 len 28672 data: 6 7 7 6 6 5 6 24/30 Read block 561 len 16384 25/30 Read block 579 len 8192 26/30 Write block 119 len 4096 data: 6 27/30 Write block 316 len 36864 data: 4 4 5 7 6 6 6 6 6 28/30 Write block 576 len 32768 data: 4 5 4 3 3 3 3 5 29/30 Write block 99 len 20480 data: 5 5 6 6 7 30/30 Write block 459 len 28672 data: 4 4 3 3 4 3 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 18 3 24 Done 15 0 15 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1011)) Some(JobId(1021)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:54.70703083Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:54.707203169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1309,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 67 67 67 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 40 40 40 66 66 66 T F T <--- 5 100-119 42 42 42 70 70 70 T F T <--- 6 120-139 44 44 44 76 72 76 F F F <--- 7 140-159 42 42 42 70 70 70 F F F 8 160-179 41 41 41 67 67 67 F F F 9 180-199 42 42 42 70 70 70 F F F 10 200-219 44 43 44 76 71 76 F F F <--- 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 43 43 43 71 71 71 T F T <--- 16 320-339 43 43 43 71 71 71 T F T <--- 17 340-359 44 44 44 73 73 73 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 44 44 44 74 74 74 T F T <--- 21 420-439 33 33 33 56 56 56 F F F 22 440-459 40 40 40 65 65 65 T F T <--- 23 460-479 41 41 41 67 67 67 T F T <--- 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 44 44 44 72 72 72 F F F 28 560-579 44 44 44 72 72 72 T F T <--- 29 580-599 38 38 38 63 63 63 T F T <--- Max gen: 44, Max flush: 76 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 67 67 67 F F F 1 020-039 37 37 37 62 62 62 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 44 44 44 76 76 76 F F F 7 140-159 42 42 42 70 70 70 F F F 8 160-179 41 41 41 67 67 67 F F F 9 180-199 42 42 42 70 70 70 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 45 45 45 77 77 77 F F F 16 320-339 45 45 45 77 77 77 F F F 17 340-359 44 44 44 73 73 73 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 45 45 45 77 77 77 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 44 44 44 72 72 72 F F F 28 560-579 45 45 45 77 77 77 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 45, Max flush: 77 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.957692814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.965925432Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966205374Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966233072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966244301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966256271Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Crucible 66699372-93a6-41f3-9f2e-9b95166735af has session id: 97fa9b19-3f0c-42ff-b22f-2cda46303929","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966593378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Upstairs opts: Upstairs UUID: 66699372-93a6-41f3-9f2e-9b95166735af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966637146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Crucible stats registered with UUID: 66699372-93a6-41f3-9f2e-9b95166735af","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966653514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:58.966663044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966776726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966792075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966802385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966950175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966965684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"66699372-93a6-41f3-9f2e-9b95166735af active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966982253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.966991042Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"0","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967000802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"1","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967010561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"2","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.96702227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.96703254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967042609Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"ds_connection connected from Ok(127.0.0.1:35241)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967106515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"ds_connection connected from Ok(127.0.0.1:45118)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967131463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"ds_connection connected from Ok(127.0.0.1:62901)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967142722Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967762822Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"0","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967794239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"1","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.967805549Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"2","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.96809007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968126588Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[0]R flush_numbers[0..12]: [67, 62, 69, 64, 77, 77, 76, 70, 67, 70, 76, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968176014Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[0]R generation[0..12]: [41, 37, 42, 39, 45, 45, 44, 42, 41, 42, 44, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968187234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968198513Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[1]R flush_numbers[0..12]: [67, 62, 69, 64, 77, 77, 76, 70, 67, 70, 76, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968209172Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[1]R generation[0..12]: [41, 37, 42, 39, 45, 45, 44, 42, 41, 42, 44, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968221621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968233031Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[2]R flush_numbers[0..12]: [67, 62, 69, 64, 77, 77, 76, 70, 67, 70, 76, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.96824358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[2]R generation[0..12]: [41, 37, 42, 39, 45, 45, 44, 42, 41, 42, 44, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968254279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968265348Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968274848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968284647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968294047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968303576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"":"downstairs","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968313795Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968322805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"0","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968332814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"1","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968342843Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"client":"2","":"downstairs","session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968353153Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968362612Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"66699372-93a6-41f3-9f2e-9b95166735af is now active with session: 97fa9b19-3f0c-42ff-b22f-2cda46303929","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968374301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968383161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"session_id":"97fa9b19-3f0c-42ff-b22f-2cda46303929"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.96839232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Activated sub_volume 66699372-93a6-41f3-9f2e-9b95166735af","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.96840034Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-18T03:29:58.968408409Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.968663542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"task":"crutest"} 01/30 Write block 456 len 40960 data: 5 3 3 5 5 4 4 5 4 5 02/30 Read block 79 len 16384 03/30 Write block 310 len 20480 data: 5 4 3 3 4 04/30 Read block 181 len 24576 05/30 Read block 495 len 36864 06/30 Write block 497 len 12288 data: 5 5 7 07/30 Read block 166 len 40960 08/30 Write block 312 len 24576 data: 4 4 5 4 5 5 09/30 Write block 484 len 28672 data: 4 5 5 6 6 5 5 10/30 Read block 67 len 8192 11/30 Read block 452 len 16384 12/30 Read block 457 len 36864 13/30 Read block 452 len 4096 14/30 Read block 15 len 32768 15/30 Read block 141 len 4096 16/30 Write block 448 len 8192 data: 5 6 17/30 Read block 97 len 4096 18/30 Read block 500 len 20480 19/30 Read block 537 len 28672 20/30 Write block 12 len 16384 data: 5 3 4 3 21/30 Write block 37 len 12288 data: 3 3 2 22/30 Flush 23/30 Write block 199 len 20480 data: 10 7 6 7 9 24/30 Read block 490 len 32768 25/30 Write block 481 len 20480 data: 5 5 4 5 6 26/30 Flush 27/30 Write block 336 len 24576 data: 4 5 6 6 7 7 28/30 Write block 531 len 12288 data: 3 3 2 29/30 Read block 153 len 32768 30/30 Read block 144 len 12288 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:29:58.977986859Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:29:58.978163467Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1317,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 41 46 78 67 78 F F F <--- 1 020-039 46 37 46 78 62 78 F F F <--- 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 44 44 44 76 76 76 F F F 7 140-159 42 42 42 70 70 70 F F F 8 160-179 41 41 41 67 67 67 F F F 9 180-199 46 42 46 79 70 79 F F F <--- 10 200-219 46 44 46 79 76 79 F F F <--- 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 46 45 46 78 77 78 F F F <--- 16 320-339 45 45 45 77 77 77 T F T <--- 17 340-359 44 44 44 73 73 73 T F T <--- 18 360-379 39 39 39 64 64 64 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 46 45 46 78 77 78 F T F <--- 23 460-479 46 45 46 78 77 78 F T F <--- 24 480-499 46 36 46 79 61 79 F F F <--- 25 500-519 33 33 33 56 56 56 F F F 26 520-539 33 33 33 56 56 56 T F T <--- 27 540-559 44 44 44 72 72 72 F F F 28 560-579 45 45 45 77 77 77 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 46, Max flush: 79 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 78 78 78 F F F 1 020-039 46 46 46 78 78 78 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 44 44 44 76 76 76 F F F 7 140-159 42 42 42 70 70 70 F F F 8 160-179 41 41 41 67 67 67 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 46 46 46 79 79 79 F F F 11 220-239 41 41 41 67 67 67 F F F 12 240-259 41 41 41 67 67 67 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 46 46 46 78 78 78 F F F 16 320-339 47 47 47 80 80 80 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 39 39 39 64 64 64 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 46 46 46 78 78 78 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 33 33 33 56 56 56 F F F 26 520-539 47 47 47 80 80 80 F F F 27 540-559 44 44 44 72 72 72 F F F 28 560-579 45 45 45 77 77 77 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 47, Max flush: 80 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.22547343Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:03.232840366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.232907341Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.240207531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.240476964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.240507722Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.240516151Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.2405279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Crucible 2795166b-cce5-4780-af83-9fc3ec204bf1 has session id: 9b29e73e-2ebd-477b-b844-ed04140a42a8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.240960022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Upstairs opts: Upstairs UUID: 2795166b-cce5-4780-af83-9fc3ec204bf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241019508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Crucible stats registered with UUID: 2795166b-cce5-4780-af83-9fc3ec204bf1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241032537Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:03.241041726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241187747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241217045Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241228054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241361785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241386274Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"2795166b-cce5-4780-af83-9fc3ec204bf1 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241402823Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241411932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"0","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241421691Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"1","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241431421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"2","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.24144104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241451239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241461159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36065)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241498756Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47561)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241509726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49890)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.241522415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242143454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"1","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242228138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"2","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242258196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"0","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242434115Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242496041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 69, 64, 77, 77, 76, 70, 67, 79, 79, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242540578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[0]R generation[0..12]: [46, 46, 42, 39, 45, 45, 44, 42, 41, 46, 46, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242552117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242563646Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 69, 64, 77, 77, 76, 70, 67, 79, 79, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242574246Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[1]R generation[0..12]: [46, 46, 42, 39, 45, 45, 44, 42, 41, 46, 46, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242584765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242595874Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 69, 64, 77, 77, 76, 70, 67, 79, 79, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242606433Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[2]R generation[0..12]: [46, 46, 42, 39, 45, 45, 44, 42, 41, 46, 46, 41]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242616923Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242631352Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242640971Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242650801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.2426601Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242669779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"":"downstairs","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242679789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242688658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"0","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242698397Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"1","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242708207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"client":"2","":"downstairs","session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242718206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242727376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"2795166b-cce5-4780-af83-9fc3ec204bf1 is now active with session: 9b29e73e-2ebd-477b-b844-ed04140a42a8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242737485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242746344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"session_id":"9b29e73e-2ebd-477b-b844-ed04140a42a8"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242755714Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Activated sub_volume 2795166b-cce5-4780-af83-9fc3ec204bf1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242764013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.242772263Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.243041365Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} 01/30 Write block 501 len 4096 data: 4 02/30 Write block 128 len 32768 data: 4 4 4 5 4 4 4 4 03/30 Write block 154 len 36864 data: 7 4 5 5 5 6 5 5 5 04/30 Write block 530 len 4096 data: 3 05/30 Write block 132 len 24576 data: 5 5 5 5 3 3 06/30 Read block 262 len 36864 07/30 Write block 488 len 28672 data: 7 6 6 4 5 6 6 08/30 Read block 305 len 16384 09/30 Read block 527 len 24576 10/30 Read block 214 len 24576 11/30 Read block 173 len 12288 12/30 Write block 135 len 20480 data: 6 4 4 3 3 13/30 Flush 14/30 Write block 535 len 20480 data: 2 2 3 3 2 15/30 Read block 536 len 4096 16/30 Read block 557 len 8192 17/30 Read block 218 len 32768 18/30 Read block 186 len 36864 19/30 Write block 476 len 40960 data: 5 4 5 5 5 6 6 5 6 7 20/30 Read block 62 len 16384 21/30 Read block 318 len 20480 22/30 Write block 147 len 12288 data: 9 9 10 23/30 Read block 590 len 40960 24/30 Read block 102 len 16384 25/30 Write block 374 len 8192 data: 3 3 26/30 Write block 240 len 4096 data: 6 27/30 Read block 394 len 40960 28/30 Write block 553 len 32768 data: 5 5 7 9 8 9 9 10 29/30 Read block 244 len 8192 30/30 Write block 215 len 36864 data: 8 7 7 7 4 4 2 3 3 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:03.25117098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:03.251353358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1328,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 78 78 78 F F F 1 020-039 46 46 46 78 78 78 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 44 48 48 76 81 81 T F F <--- 7 140-159 42 48 48 70 81 81 F T T <--- 8 160-179 41 48 48 67 81 81 F F F <--- 9 180-199 46 46 46 79 79 79 F F F 10 200-219 46 46 46 79 79 79 F T T <--- 11 220-239 41 41 41 67 67 67 F T T <--- 12 240-259 41 41 41 67 67 67 F T T <--- 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 46 46 46 78 78 78 F F F 16 320-339 47 47 47 80 80 80 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 39 39 39 64 64 64 F T T <--- 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 46 46 46 78 78 78 F T T <--- 24 480-499 46 48 48 79 81 81 F T T <--- 25 500-519 33 48 48 56 81 81 T F F <--- 26 520-539 47 48 48 80 81 81 F T T <--- 27 540-559 44 44 44 72 72 72 F T T <--- 28 560-579 45 45 45 77 77 77 F T T <--- 29 580-599 45 45 45 77 77 77 F F F Max gen: 48, Max flush: 81 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 78 78 78 F F F 1 020-039 46 46 46 78 78 78 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 39 39 39 64 64 64 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 49 49 49 82 82 82 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 49 49 49 82 82 82 F F F 11 220-239 49 49 49 82 82 82 F F F 12 240-259 49 49 49 82 82 82 F F F 13 260-279 38 38 38 63 63 63 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 46 46 46 78 78 78 F F F 16 320-339 47 47 47 80 80 80 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 49 49 49 82 82 82 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 33 33 33 56 56 56 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 49 49 49 82 82 82 F F F 24 480-499 49 49 49 82 82 82 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 49 49 49 82 82 82 F F F 27 540-559 49 49 49 82 82 82 F F F 28 560-579 49 49 49 82 82 82 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 49, Max flush: 82 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.511907239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520274389Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.52055468Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520581928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520590508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520602207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Crucible 0fe7b671-558e-45c5-aaf8-64f98206bcc5 has session id: 5470b545-25d9-4d9d-b9bd-6baf35ae882d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520926766Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Upstairs opts: Upstairs UUID: 0fe7b671-558e-45c5-aaf8-64f98206bcc5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520950864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Crucible stats registered with UUID: 0fe7b671-558e-45c5-aaf8-64f98206bcc5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.520961004Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:07.520970133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521126253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521156771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.52116781Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521300831Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.52132046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"0fe7b671-558e-45c5-aaf8-64f98206bcc5 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521349278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521358777Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"0","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521368727Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"1","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521378386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"2","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521388055Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521398175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521408204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54323)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521502478Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52210)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521533876Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59316)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.521545315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522162684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"0","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522190333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"1","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522290976Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"2","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522455985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522472514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 69, 64, 77, 77, 81, 82, 81, 79, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522567258Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[0]R generation[0..12]: [46, 46, 42, 39, 45, 45, 48, 49, 48, 46, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522578387Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522589816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 69, 64, 77, 77, 81, 82, 81, 79, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522600336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[1]R generation[0..12]: [46, 46, 42, 39, 45, 45, 48, 49, 48, 46, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522610745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522621854Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 69, 64, 77, 77, 81, 82, 81, 79, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522632474Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[2]R generation[0..12]: [46, 46, 42, 39, 45, 45, 48, 49, 48, 46, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522642883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522653862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522663312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522673031Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.52268226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.52269189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"":"downstairs","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522701599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522712058Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"0","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522722138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"1","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522732147Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"client":"2","":"downstairs","session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522742226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522751536Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"0fe7b671-558e-45c5-aaf8-64f98206bcc5 is now active with session: 5470b545-25d9-4d9d-b9bd-6baf35ae882d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522761635Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522770335Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"session_id":"5470b545-25d9-4d9d-b9bd-6baf35ae882d"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522779404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Activated sub_volume 0fe7b671-558e-45c5-aaf8-64f98206bcc5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522787383Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.522795553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.523095603Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"task":"crutest"} 01/30 Write block 19 len 40960 data: 4 5 5 5 5 5 4 4 3 3 02/30 Read block 555 len 32768 03/30 Read block 505 len 40960 04/30 Write block 69 len 36864 data: 5 4 4 5 6 6 6 6 5 05/30 Write block 563 len 40960 data: 5 4 3 4 4 4 4 5 5 4 06/30 Write block 341 len 36864 data: 8 7 7 6 6 6 6 4 4 07/30 Read block 327 len 24576 08/30 Read block 525 len 12288 09/30 Flush 10/30 Write block 276 len 16384 data: 4 2 2 2 11/30 Write block 214 len 8192 data: 10 9 12/30 Read block 125 len 36864 13/30 Flush 14/30 Write block 264 len 24576 data: 2 2 3 3 4 3 15/30 Write block 447 len 8192 data: 5 6 16/30 Flush 17/30 Write block 321 len 24576 data: 7 7 7 7 5 5 18/30 Write block 262 len 40960 data: 3 2 3 3 4 4 5 4 3 3 19/30 Write block 326 len 12288 data: 6 5 6 20/30 Flush 21/30 Write block 376 len 4096 data: 2 22/30 Read block 84 len 24576 23/30 Read block 568 len 32768 24/30 Write block 434 len 32768 data: 3 4 4 4 5 4 4 4 25/30 Write block 332 len 28672 data: 4 4 2 3 5 6 7 26/30 Read block 21 len 16384 27/30 Write block 530 len 28672 data: 4 4 4 3 2 3 3 28/30 Write block 173 len 36864 data: 6 5 4 3 4 4 5 5 7 29/30 Write block 70 len 4096 data: 5 30/30 Read block 87 len 40960 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:07.532152337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:07.532330466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1336,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 46 50 83 78 83 F T F <--- 1 020-039 50 46 50 83 78 83 F T F <--- 2 040-059 42 42 42 69 69 69 F F F 3 060-079 50 39 50 83 64 83 T F T <--- 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 49 49 49 82 82 82 F F F 8 160-179 48 48 48 81 81 81 T F T <--- 9 180-199 46 46 46 79 79 79 T F T <--- 10 200-219 50 49 50 84 82 84 F F F <--- 11 220-239 49 49 49 82 82 82 F F F 12 240-259 49 49 49 82 82 82 F F F 13 260-279 50 38 50 86 63 86 F F F <--- 14 280-299 42 42 42 70 70 70 F F F 15 300-319 46 46 46 78 78 78 F F F 16 320-339 50 47 50 86 80 86 T F T <--- 17 340-359 50 47 50 83 80 83 F F F <--- 18 360-379 49 49 49 82 82 82 T F T <--- 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 33 33 33 56 56 56 T F T <--- 22 440-459 50 46 50 85 78 85 T F T <--- 23 460-479 49 49 49 82 82 82 F F F 24 480-499 49 49 49 82 82 82 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 49 49 49 82 82 82 T F T <--- 27 540-559 49 49 49 82 82 82 F F F 28 560-579 50 49 50 83 82 83 F F F <--- 29 580-599 45 45 45 77 77 77 F F F Max gen: 50, Max flush: 86 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 83 83 83 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 51 51 51 87 87 87 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F F F 6 120-139 48 48 48 81 81 81 F F F 7 140-159 49 49 49 82 82 82 F F F 8 160-179 51 51 51 87 87 87 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 50 50 50 84 84 84 F F F 11 220-239 49 49 49 82 82 82 F F F 12 240-259 49 49 49 82 82 82 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 42 42 42 70 70 70 F F F 15 300-319 46 46 46 78 78 78 F F F 16 320-339 51 51 51 87 87 87 F F F 17 340-359 50 50 50 83 83 83 F F F 18 360-379 51 51 51 87 87 87 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F F F 21 420-439 51 51 51 87 87 87 F F F 22 440-459 51 51 51 87 87 87 F F F 23 460-479 49 49 49 82 82 82 F F F 24 480-499 49 49 49 82 82 82 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 51 51 51 87 87 87 F F F 27 540-559 49 49 49 82 82 82 F F F 28 560-579 50 50 50 83 83 83 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 51, Max flush: 87 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.794385719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:11.80166667Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.801723686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.808782022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809043605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809069393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809080562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809092392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Crucible 4b22418f-45da-48ed-a145-3a54e4c8622b has session id: a02ffc8e-782a-4224-b75a-b68e924e48e7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809451748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Upstairs opts: Upstairs UUID: 4b22418f-45da-48ed-a145-3a54e4c8622b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809472197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Crucible stats registered with UUID: 4b22418f-45da-48ed-a145-3a54e4c8622b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809482036Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:11.809491135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809647265Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809677373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809688232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809814424Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809829573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"4b22418f-45da-48ed-a145-3a54e4c8622b active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809864711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.80987404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"0","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809883809Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"1","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809895899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"2","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809905568Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809934576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809948045Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56711)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809958715Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52882)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.809969244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53350)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.81002668Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.810615411Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"1","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.810659938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"2","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.810747103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"0","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.810971218Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.810981087Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[0]R flush_numbers[0..12]: [83, 83, 69, 87, 77, 77, 81, 82, 87, 87, 84, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811072221Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[0]R generation[0..12]: [50, 50, 42, 51, 45, 45, 48, 49, 51, 51, 50, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811083301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.81109451Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[1]R flush_numbers[0..12]: [83, 83, 69, 87, 77, 77, 81, 82, 87, 87, 84, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811107789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[1]R generation[0..12]: [50, 50, 42, 51, 45, 45, 48, 49, 51, 51, 50, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811118308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811129188Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[2]R flush_numbers[0..12]: [83, 83, 69, 87, 77, 77, 81, 82, 87, 87, 84, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811139557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[2]R generation[0..12]: [50, 50, 42, 51, 45, 45, 48, 49, 51, 51, 50, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811149986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811160875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811170265Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811179944Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811189244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811198613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"":"downstairs","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811208332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811217282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"0","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811227011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"1","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.81123674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"client":"2","":"downstairs","session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.81124669Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811257669Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"4b22418f-45da-48ed-a145-3a54e4c8622b is now active with session: a02ffc8e-782a-4224-b75a-b68e924e48e7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811267788Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811276608Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"session_id":"a02ffc8e-782a-4224-b75a-b68e924e48e7"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811285687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Activated sub_volume 4b22418f-45da-48ed-a145-3a54e4c8622b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811293667Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811301816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.811613356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} 01/30 Write block 327 len 28672 data: 6 7 6 5 4 5 5 02/30 Read block 170 len 12288 03/30 Read block 122 len 36864 04/30 Read block 251 len 40960 05/30 Write block 288 len 32768 data: 6 5 6 6 4 4 6 6 06/30 Read block 376 len 32768 07/30 Read block 88 len 36864 08/30 Write block 588 len 28672 data: 4 4 5 5 5 4 5 09/30 Write block 367 len 24576 data: 7 7 6 5 4 4 10/30 Read block 14 len 16384 11/30 Read block 446 len 32768 12/30 Read block 418 len 16384 13/30 Read block 231 len 16384 14/30 Flush 15/30 Write block 567 len 4096 data: 5 16/30 Read block 15 len 36864 17/30 Write block 421 len 16384 data: 4 6 4 5 18/30 Read block 116 len 36864 19/30 Read block 439 len 12288 20/30 Read block 58 len 8192 21/30 Read block 595 len 4096 22/30 Write block 192 len 8192 data: 2 3 23/30 Read block 346 len 40960 24/30 Read block 571 len 20480 25/30 Write block 466 len 16384 data: 3 4 3 4 26/30 Read block 11 len 12288 27/30 Write block 488 len 12288 data: 8 7 7 28/30 Write block 407 len 4096 data: 6 29/30 Write block 118 len 28672 data: 5 7 7 7 7 7 7 30/30 Write block 215 len 12288 data: 10 8 8 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 3 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 4 4 37 Done 1 26 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:11.82006029Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:11.820197601Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1344,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 83 83 83 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 51 51 51 87 87 87 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 45 45 45 77 77 77 F T T <--- 6 120-139 48 48 48 81 81 81 F T T <--- 7 140-159 49 49 49 82 82 82 F F F 8 160-179 51 51 51 87 87 87 F F F 9 180-199 51 51 51 87 87 87 F T T <--- 10 200-219 50 50 50 84 84 84 F T T <--- 11 220-239 49 49 49 82 82 82 F F F 12 240-259 49 49 49 82 82 82 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 42 52 52 70 88 88 F F F <--- 15 300-319 46 46 46 78 78 78 F F F 16 320-339 51 52 52 87 88 88 T F F <--- 17 340-359 50 50 50 83 83 83 F F F 18 360-379 51 52 52 87 88 88 F F F <--- 19 380-399 43 43 43 71 71 71 F F F 20 400-419 45 45 45 77 77 77 F T T <--- 21 420-439 51 51 51 87 87 87 F T T <--- 22 440-459 51 51 51 87 87 87 F F F 23 460-479 49 49 49 82 82 82 F T T <--- 24 480-499 49 49 49 82 82 82 F T T <--- 25 500-519 48 48 48 81 81 81 F F F 26 520-539 51 51 51 87 87 87 F F F 27 540-559 49 49 49 82 82 82 F F F 28 560-579 50 50 50 83 83 83 F T T <--- 29 580-599 45 52 52 77 88 88 F F F <--- Max gen: 52, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 83 83 83 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 51 51 51 87 87 87 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 53 53 53 89 89 89 F F F 6 120-139 53 53 53 89 89 89 F F F 7 140-159 49 49 49 82 82 82 F F F 8 160-179 51 51 51 87 87 87 F F F 9 180-199 53 53 53 89 89 89 F F F 10 200-219 53 53 53 89 89 89 F F F 11 220-239 49 49 49 82 82 82 F F F 12 240-259 49 49 49 82 82 82 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 46 46 46 78 78 78 F F F 16 320-339 52 52 52 88 88 88 F F F 17 340-359 50 50 50 83 83 83 F F F 18 360-379 52 52 52 88 88 88 F F F 19 380-399 43 43 43 71 71 71 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 53 53 53 89 89 89 F F F 22 440-459 51 51 51 87 87 87 F F F 23 460-479 53 53 53 89 89 89 F F F 24 480-499 53 53 53 89 89 89 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 51 51 51 87 87 87 F F F 27 540-559 49 49 49 82 82 82 F F F 28 560-579 53 53 53 89 89 89 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 53, Max flush: 89 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.082404493Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.090779662Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091053294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091083712Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091092362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091104291Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Crucible 248c4f35-a98e-414c-ad56-a008ffd78aaa has session id: dc622dc7-3890-4436-bf75-29012bcbc1d0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091458768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Upstairs opts: Upstairs UUID: 248c4f35-a98e-414c-ad56-a008ffd78aaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091489616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Crucible stats registered with UUID: 248c4f35-a98e-414c-ad56-a008ffd78aaa","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091503385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:16.091512664Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091627797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091657655Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091668574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091792256Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091806605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"248c4f35-a98e-414c-ad56-a008ffd78aaa active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091823494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091832373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"0","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091842133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"1","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091851762Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"2","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091861291Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091871451Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.09188154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61769)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.091935126Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55816)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.092062238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52650)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.092073627Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.092627111Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"2","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.09264454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"0","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.092655969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"1","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.092958169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.092980378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[0]R flush_numbers[0..12]: [83, 83, 69, 87, 77, 89, 89, 82, 87, 89, 89, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093020805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[0]R generation[0..12]: [50, 50, 42, 51, 45, 53, 53, 49, 51, 53, 53, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093032304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093043514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[1]R flush_numbers[0..12]: [83, 83, 69, 87, 77, 89, 89, 82, 87, 89, 89, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093053913Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[1]R generation[0..12]: [50, 50, 42, 51, 45, 53, 53, 49, 51, 53, 53, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093064632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093075641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[2]R flush_numbers[0..12]: [83, 83, 69, 87, 77, 89, 89, 82, 87, 89, 89, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093086111Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[2]R generation[0..12]: [50, 50, 42, 51, 45, 53, 53, 49, 51, 53, 53, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.09309638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093107659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093120378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093130128Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093139497Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093148927Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"":"downstairs","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093158526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093167225Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"0","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093177015Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"1","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093186594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"client":"2","":"downstairs","session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093196323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093205593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"248c4f35-a98e-414c-ad56-a008ffd78aaa is now active with session: dc622dc7-3890-4436-bf75-29012bcbc1d0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093215582Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093224322Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"session_id":"dc622dc7-3890-4436-bf75-29012bcbc1d0"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.093233401Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Activated sub_volume 248c4f35-a98e-414c-ad56-a008ffd78aaa","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.09324151Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.09324964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.0935579Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"task":"crutest"} 01/30 Write block 344 len 36864 data: 7 7 7 7 5 5 5 5 5 02/30 Read block 100 len 12288 03/30 Read block 550 len 24576 04/30 Write block 543 len 4096 data: 4 05/30 Write block 180 len 24576 data: 6 8 7 6 7 5 06/30 Write block 363 len 32768 data: 8 9 7 8 8 8 7 6 07/30 Write block 389 len 16384 data: 5 6 7 8 08/30 Write block 137 len 28672 data: 5 4 4 3 6 9 8 09/30 Read block 179 len 40960 10/30 Write block 96 len 8192 data: 3 3 11/30 Read block 11 len 4096 12/30 Read block 492 len 16384 13/30 Read block 524 len 20480 14/30 Write block 200 len 24576 data: 8 7 8 10 10 10 15/30 Write block 100 len 32768 data: 6 7 7 8 6 7 4 5 16/30 Read block 395 len 40960 17/30 Read block 491 len 16384 18/30 Read block 18 len 4096 19/30 Read block 30 len 12288 20/30 Flush 21/30 Write block 221 len 12288 data: 3 4 4 22/30 Write block 245 len 16384 data: 6 6 7 6 23/30 Read block 117 len 16384 24/30 Write block 536 len 4096 data: 4 25/30 Read block 98 len 8192 26/30 Flush 27/30 Write block 305 len 28672 data: 5 5 5 5 4 6 5 28/30 Write block 549 len 12288 data: 8 8 7 29/30 Write block 127 len 12288 data: 5 5 5 30/30 Write block 572 len 20480 data: 5 3 3 4 5 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 7 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 25 4 33 Done 26 5 26 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:16.101350227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:16.101500197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1352,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 83 83 83 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 51 51 51 87 87 87 F F F 4 080-099 54 45 54 90 77 90 F F F <--- 5 100-119 54 53 54 90 89 90 F F F <--- 6 120-139 54 53 54 90 89 90 T F T <--- 7 140-159 54 49 54 90 82 90 F F F <--- 8 160-179 51 51 51 87 87 87 F F F 9 180-199 54 53 54 90 89 90 F T F <--- 10 200-219 54 53 54 90 89 90 F F F <--- 11 220-239 54 49 54 91 82 91 F F F <--- 12 240-259 54 49 54 91 82 91 F F F <--- 13 260-279 50 50 50 86 86 86 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 46 46 46 78 78 78 T F T <--- 16 320-339 52 52 52 88 88 88 F F F 17 340-359 54 50 54 90 83 90 F T F <--- 18 360-379 54 52 54 90 88 90 F F F <--- 19 380-399 54 43 54 90 71 90 F F F <--- 20 400-419 53 53 53 89 89 89 F F F 21 420-439 53 53 53 89 89 89 F F F 22 440-459 51 51 51 87 87 87 F F F 23 460-479 53 53 53 89 89 89 F F F 24 480-499 53 53 53 89 89 89 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 54 51 54 91 87 91 F F F <--- 27 540-559 54 49 54 90 82 90 T T T <--- 28 560-579 53 53 53 89 89 89 T F T <--- 29 580-599 52 52 52 88 88 88 F F F Max gen: 54, Max flush: 91 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 83 83 83 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 51 51 51 87 87 87 F F F 4 080-099 54 54 54 90 90 90 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 55 55 55 92 92 92 F F F 7 140-159 54 54 54 90 90 90 F F F 8 160-179 51 51 51 87 87 87 F F F 9 180-199 54 54 54 90 90 90 F F F 10 200-219 54 54 54 90 90 90 F F F 11 220-239 54 54 54 91 91 91 F F F 12 240-259 54 54 54 91 91 91 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 92 92 92 F F F 16 320-339 52 52 52 88 88 88 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 53 53 53 89 89 89 F F F 22 440-459 51 51 51 87 87 87 F F F 23 460-479 53 53 53 89 89 89 F F F 24 480-499 53 53 53 89 89 89 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 54 54 54 91 91 91 F F F 27 540-559 55 55 55 92 92 92 F F F 28 560-579 55 55 55 92 92 92 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 55, Max flush: 92 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.356592567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.364875773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365160164Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365186652Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365195182Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365206951Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Crucible 1519e198-3ca5-4c12-b1dc-b57f6fc27a0b has session id: 2c60b393-ba6d-4752-a3b1-31cd9abafab1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365596935Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Upstairs opts: Upstairs UUID: 1519e198-3ca5-4c12-b1dc-b57f6fc27a0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365619004Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Crucible stats registered with UUID: 1519e198-3ca5-4c12-b1dc-b57f6fc27a0b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365628843Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:20.365637882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365784023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365799032Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365809071Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.365965731Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.36598228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"1519e198-3ca5-4c12-b1dc-b57f6fc27a0b active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366007968Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366017597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"0","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366027387Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"1","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366037046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"2","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366046666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366056755Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366066864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61249)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.36612912Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49010)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366155378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62835)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366244533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366801396Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"0","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366823484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"2","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.366955306Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"1","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367109256Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367124855Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[0]R flush_numbers[0..12]: [83, 83, 69, 87, 90, 90, 92, 90, 87, 90, 90, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367276485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[0]R generation[0..12]: [50, 50, 42, 51, 54, 54, 55, 54, 51, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367287884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367299113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[1]R flush_numbers[0..12]: [83, 83, 69, 87, 90, 90, 92, 90, 87, 90, 90, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367309573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[1]R generation[0..12]: [50, 50, 42, 51, 54, 54, 55, 54, 51, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367320092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367330981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[2]R flush_numbers[0..12]: [83, 83, 69, 87, 90, 90, 92, 90, 87, 90, 90, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.36734156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[2]R generation[0..12]: [50, 50, 42, 51, 54, 54, 55, 54, 51, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.36735209Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367363089Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367372518Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367382248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367391497Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367400827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"":"downstairs","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367410516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367419355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"0","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367430795Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"1","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367440514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"client":"2","":"downstairs","session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367450383Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367459493Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"1519e198-3ca5-4c12-b1dc-b57f6fc27a0b is now active with session: 2c60b393-ba6d-4752-a3b1-31cd9abafab1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367469572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367478211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"session_id":"2c60b393-ba6d-4752-a3b1-31cd9abafab1"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367487241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Activated sub_volume 1519e198-3ca5-4c12-b1dc-b57f6fc27a0b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.36749522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.36750333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.367796161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"task":"crutest"} 01/30 Write block 29 len 20480 data: 3 3 3 3 3 02/30 Read block 50 len 8192 03/30 Read block 11 len 20480 04/30 Read block 178 len 12288 05/30 Read block 201 len 20480 06/30 Read block 347 len 20480 07/30 Write block 492 len 40960 data: 6 7 7 5 5 6 6 8 5 5 08/30 Read block 117 len 32768 09/30 Write block 157 len 12288 data: 6 6 7 10/30 Read block 396 len 32768 11/30 Read block 184 len 24576 12/30 Read block 545 len 12288 13/30 Read block 400 len 28672 14/30 Read block 343 len 24576 15/30 Read block 123 len 12288 16/30 Read block 136 len 28672 17/30 Flush 18/30 Write block 8 len 4096 data: 8 19/30 Flush 20/30 Write block 63 len 28672 data: 4 3 4 2 2 2 6 21/30 Read block 524 len 20480 22/30 Read block 96 len 20480 23/30 Read block 562 len 24576 24/30 Write block 98 len 16384 data: 5 6 7 8 25/30 Write block 14 len 32768 data: 5 4 3 4 4 5 6 6 26/30 Write block 214 len 12288 data: 11 11 9 27/30 Write block 211 len 12288 data: 13 13 12 28/30 Write block 566 len 16384 data: 5 6 5 5 29/30 Read block 269 len 12288 30/30 Write block 97 len 32768 data: 4 6 7 8 9 8 9 7 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 2 Done Sent Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:20.376512007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:20.376683146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1360,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 50 56 94 83 94 T F T <--- 1 020-039 56 50 56 93 83 93 T F T <--- 2 040-059 42 42 42 69 69 69 F F F 3 060-079 51 51 51 87 87 87 T F T <--- 4 080-099 54 54 54 90 90 90 T F T <--- 5 100-119 54 54 54 90 90 90 T F T <--- 6 120-139 55 55 55 92 92 92 F F F 7 140-159 56 54 56 93 90 93 F F F <--- 8 160-179 51 51 51 87 87 87 F F F 9 180-199 54 54 54 90 90 90 F F F 10 200-219 54 54 54 90 90 90 T F T <--- 11 220-239 54 54 54 91 91 91 F F F 12 240-259 54 54 54 91 91 91 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 92 92 92 F F F 16 320-339 52 52 52 88 88 88 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 53 53 53 89 89 89 F F F 22 440-459 51 51 51 87 87 87 F F F 23 460-479 53 53 53 89 89 89 F F F 24 480-499 56 53 56 93 89 93 F F F <--- 25 500-519 56 48 56 93 81 93 F F F <--- 26 520-539 54 54 54 91 91 91 F F F 27 540-559 55 55 55 92 92 92 F F F 28 560-579 55 55 55 92 92 92 T F T <--- 29 580-599 52 52 52 88 88 88 F F F Max gen: 56, Max flush: 94 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 95 95 95 F F F 1 020-039 57 57 57 95 95 95 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 57 57 57 95 95 95 F F F 4 080-099 57 57 57 95 95 95 F F F 5 100-119 57 57 57 95 95 95 F F F 6 120-139 55 55 55 92 92 92 F F F 7 140-159 56 56 56 93 93 93 F F F 8 160-179 51 51 51 87 87 87 F F F 9 180-199 54 54 54 90 90 90 F F F 10 200-219 57 57 57 95 95 95 F F F 11 220-239 54 54 54 91 91 91 F F F 12 240-259 54 54 54 91 91 91 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 92 92 92 F F F 16 320-339 52 52 52 88 88 88 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 53 53 53 89 89 89 F F F 22 440-459 51 51 51 87 87 87 F F F 23 460-479 53 53 53 89 89 89 F F F 24 480-499 56 56 56 93 93 93 F F F 25 500-519 56 56 56 93 93 93 F F F 26 520-539 54 54 54 91 91 91 F F F 27 540-559 55 55 55 92 92 92 F F F 28 560-579 57 57 57 95 95 95 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 57, Max flush: 95 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.622022708Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.630247817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.630529199Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.630554287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.630562797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.630574616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Crucible 3c3c9b25-38f7-46bd-82b0-950e5012ad58 has session id: 44856ce2-8157-4f6d-a2b7-124ce0bb4a24","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.630931133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Upstairs opts: Upstairs UUID: 3c3c9b25-38f7-46bd-82b0-950e5012ad58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63096288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Crucible stats registered with UUID: 3c3c9b25-38f7-46bd-82b0-950e5012ad58","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63097449Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:24.630983859Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631107541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.6311233Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631133719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631293309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631317257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"3c3c9b25-38f7-46bd-82b0-950e5012ad58 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631340996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631350365Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"0","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631360184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"1","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631369964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"2","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631379563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631389562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631399522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"ds_connection connected from Ok(127.0.0.1:57981)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631494366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"ds_connection connected from Ok(127.0.0.1:60297)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631586039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"ds_connection connected from Ok(127.0.0.1:44032)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.631597969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63217911Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"2","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632206539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"1","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632222878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"0","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63249041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632521088Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[0]R flush_numbers[0..12]: [95, 95, 69, 95, 95, 95, 92, 93, 87, 90, 95, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632570825Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[0]R generation[0..12]: [57, 57, 42, 57, 57, 57, 55, 56, 51, 54, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632582144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632593323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[1]R flush_numbers[0..12]: [95, 95, 69, 95, 95, 95, 92, 93, 87, 90, 95, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632603753Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[1]R generation[0..12]: [57, 57, 42, 57, 57, 57, 55, 56, 51, 54, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632614432Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632625561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[2]R flush_numbers[0..12]: [95, 95, 69, 95, 95, 95, 92, 93, 87, 90, 95, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63263881Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[2]R generation[0..12]: [57, 57, 42, 57, 57, 57, 55, 56, 51, 54, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63264945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632660539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632670138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632679868Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632689227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632698706Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"":"downstairs","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632708846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632717775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"0","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632727844Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"1","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632737824Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"client":"2","":"downstairs","session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632747963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632757252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"3c3c9b25-38f7-46bd-82b0-950e5012ad58 is now active with session: 44856ce2-8157-4f6d-a2b7-124ce0bb4a24","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632767392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632776151Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"session_id":"44856ce2-8157-4f6d-a2b7-124ce0bb4a24"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632785191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Activated sub_volume 3c3c9b25-38f7-46bd-82b0-950e5012ad58","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.63279494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.632803069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.633062152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"task":"crutest"} 01/30 Write block 329 len 36864 data: 7 6 5 6 6 3 4 6 7 02/30 Read block 348 len 8192 03/30 Read block 520 len 36864 04/30 Read block 502 len 12288 05/30 Write block 156 len 24576 data: 6 7 7 8 6 6 06/30 Write block 539 len 28672 data: 3 4 4 4 5 4 5 07/30 Read block 337 len 12288 08/30 Read block 249 len 24576 09/30 Read block 561 len 24576 10/30 Read block 236 len 24576 11/30 Write block 236 len 24576 data: 5 6 6 7 7 5 12/30 Read block 149 len 16384 13/30 Write block 437 len 36864 data: 5 6 5 5 5 4 4 3 4 14/30 Write block 177 len 12288 data: 5 5 6 15/30 Read block 40 len 36864 16/30 Write block 261 len 4096 data: 3 17/30 Write block 89 len 12288 data: 3 4 4 18/30 Read block 3 len 24576 19/30 Write block 42 len 12288 data: 2 3 4 20/30 Read block 121 len 24576 21/30 Read block 169 len 8192 22/30 Read block 233 len 16384 23/30 Write block 366 len 40960 data: 9 9 9 8 7 5 5 4 4 4 24/30 Read block 567 len 40960 25/30 Read block 432 len 40960 26/30 Read block 409 len 12288 27/30 Write block 134 len 20480 data: 6 7 5 6 5 28/30 Read block 106 len 32768 29/30 Read block 210 len 4096 30/30 Read block 594 len 4096 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:24.642304544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:24.642475103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1368,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 95 95 95 F F F 1 020-039 57 57 57 95 95 95 F F F 2 040-059 42 42 42 69 69 69 T F T <--- 3 060-079 57 57 57 95 95 95 F F F 4 080-099 57 57 57 95 95 95 T F T <--- 5 100-119 57 57 57 95 95 95 F F F 6 120-139 55 55 55 92 92 92 T F T <--- 7 140-159 56 56 56 93 93 93 T F T <--- 8 160-179 51 51 51 87 87 87 T F T <--- 9 180-199 54 54 54 90 90 90 F F F 10 200-219 57 57 57 95 95 95 F F F 11 220-239 54 54 54 91 91 91 T F T <--- 12 240-259 54 54 54 91 91 91 T F T <--- 13 260-279 50 50 50 86 86 86 T F T <--- 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 92 92 92 F F F 16 320-339 52 52 52 88 88 88 T T T 17 340-359 54 54 54 90 90 90 F F F 18 360-379 54 54 54 90 90 90 T F T <--- 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 53 53 53 89 89 89 T F T <--- 22 440-459 51 51 51 87 87 87 T F T <--- 23 460-479 53 53 53 89 89 89 F F F 24 480-499 56 56 56 93 93 93 F F F 25 500-519 56 56 56 93 93 93 F F F 26 520-539 54 54 54 91 91 91 T F T <--- 27 540-559 55 55 55 92 92 92 T F T <--- 28 560-579 57 57 57 95 95 95 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 57, Max flush: 95 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 95 95 95 F F F 1 020-039 57 57 57 95 95 95 F F F 2 040-059 58 58 58 96 96 96 F F F 3 060-079 57 57 57 95 95 95 F F F 4 080-099 58 58 58 96 96 96 F F F 5 100-119 57 57 57 95 95 95 F F F 6 120-139 58 58 58 96 96 96 F F F 7 140-159 58 58 58 96 96 96 F F F 8 160-179 58 58 58 96 96 96 F F F 9 180-199 54 54 54 90 90 90 F F F 10 200-219 57 57 57 95 95 95 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 58 58 58 96 96 96 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 92 92 92 F F F 16 320-339 58 58 58 96 96 96 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 58 58 58 96 96 96 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 53 53 53 89 89 89 F F F 24 480-499 56 56 56 93 93 93 F F F 25 500-519 56 56 56 93 93 93 F F F 26 520-539 58 58 58 96 96 96 F F F 27 540-559 58 58 58 96 96 96 F F F 28 560-579 57 57 57 95 95 95 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 58, Max flush: 96 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.904871643Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.913187746Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.913464428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.913492716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.913501425Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.913513305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Crucible 43b7ddfe-5b3c-4207-b1a2-4c50c5d0c6bb has session id: 48922fe2-2d98-40b5-a9f2-e597854d3f13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.913852312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Upstairs opts: Upstairs UUID: 43b7ddfe-5b3c-4207-b1a2-4c50c5d0c6bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.91388234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Crucible stats registered with UUID: 43b7ddfe-5b3c-4207-b1a2-4c50c5d0c6bb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.91389377Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:28.913903059Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914025461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.91404061Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914053089Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914207339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914222828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"43b7ddfe-5b3c-4207-b1a2-4c50c5d0c6bb active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914239597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914248506Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"0","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914258166Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"1","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914267925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"2","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914277584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914287644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914297503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"ds_connection connected from Ok(127.0.0.1:61757)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914375558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"ds_connection connected from Ok(127.0.0.1:61908)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914400866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"ds_connection connected from Ok(127.0.0.1:61048)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.914508389Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915025495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"0","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915043494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"1","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915054733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"2","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915355193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915365763Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[0]R flush_numbers[0..12]: [95, 95, 96, 95, 96, 95, 96, 96, 96, 90, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915390811Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[0]R generation[0..12]: [57, 57, 58, 57, 58, 57, 58, 58, 58, 54, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.91540203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915424439Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[1]R flush_numbers[0..12]: [95, 95, 96, 95, 96, 95, 96, 96, 96, 90, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915435208Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[1]R generation[0..12]: [57, 57, 58, 57, 58, 57, 58, 58, 58, 54, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915445638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915456687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[2]R flush_numbers[0..12]: [95, 95, 96, 95, 96, 95, 96, 96, 96, 90, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915467096Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[2]R generation[0..12]: [57, 57, 58, 57, 58, 57, 58, 58, 58, 54, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915477615Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915488545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Max found gen is 59","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915497894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Generation requested: 60 >= found:59","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915507653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915517033Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915528442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"":"downstairs","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915538381Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915547151Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"0","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.91555689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"1","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.91556674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"client":"2","":"downstairs","session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915576679Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915585818Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"43b7ddfe-5b3c-4207-b1a2-4c50c5d0c6bb is now active with session: 48922fe2-2d98-40b5-a9f2-e597854d3f13Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915601297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915610217Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"session_id":"48922fe2-2d98-40b5-a9f2-e597854d3f13"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915619526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Activated sub_volume 43b7ddfe-5b3c-4207-b1a2-4c50c5d0c6bb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915627726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915635975Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.915840112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"task":"crutest"} 01/30 Write block 37 len 24576 data: 4 4 3 2 2 3 02/30 Read block 277 len 20480 03/30 Write block 176 len 28672 data: 4 6 6 7 7 9 8 04/30 Read block 441 len 12288 05/30 Write block 562 len 40960 data: 8 6 5 4 6 7 6 6 6 6 06/30 Read block 212 len 4096 07/30 Write block 524 len 16384 data: 2 3 3 3 08/30 Write block 546 len 12288 data: 5 8 7 09/30 Write block 145 len 36864 data: 6 7 10 10 11 9 9 9 7 10/30 Read block 340 len 16384 11/30 Write block 0 len 12288 data: 4 5 5 12/30 Write block 188 len 12288 data: 5 4 4 13/30 Write block 187 len 32768 data: 5 6 5 5 2 3 4 4 14/30 Write block 104 len 28672 data: 8 8 5 6 6 5 5 15/30 Flush 16/30 Write block 436 len 16384 data: 5 6 7 6 17/30 Read block 283 len 4096 18/30 Flush 19/30 Flush 20/30 Write block 300 len 24576 data: 7 6 6 5 6 6 21/30 Write block 462 len 20480 data: 5 6 5 6 4 22/30 Read block 196 len 20480 23/30 Read block 315 len 12288 24/30 Read block 295 len 40960 25/30 Read block 456 len 12288 26/30 Read block 266 len 28672 27/30 Write block 93 len 40960 data: 3 3 3 4 5 7 8 9 10 9 28/30 Read block 395 len 28672 29/30 Write block 248 len 36864 data: 7 6 7 6 5 5 4 3 3 30/30 Read block 189 len 12288 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:28.92528959Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:28.925495277Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1376,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 57 60 97 95 97 F F F <--- 1 020-039 60 57 60 97 95 97 F T F <--- 2 040-059 60 58 60 97 96 97 F T F <--- 3 060-079 57 57 57 95 95 95 F F F 4 080-099 58 58 58 96 96 96 T F T <--- 5 100-119 60 57 60 97 95 97 T F T <--- 6 120-139 58 58 58 96 96 96 F F F 7 140-159 60 58 60 97 96 97 F F F <--- 8 160-179 60 58 60 97 96 97 F F F <--- 9 180-199 60 54 60 97 90 97 F F F <--- 10 200-219 57 57 57 95 95 95 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 58 58 58 96 96 96 T F T <--- 13 260-279 58 58 58 96 96 96 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 92 92 92 T F T <--- 16 320-339 58 58 58 96 96 96 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 60 58 60 98 96 98 F F F <--- 22 440-459 58 58 58 96 96 96 F F F 23 460-479 53 53 53 89 89 89 T F T <--- 24 480-499 56 56 56 93 93 93 F F F 25 500-519 56 56 56 93 93 93 F F F 26 520-539 60 58 60 97 96 97 F F F <--- 27 540-559 60 58 60 97 96 97 F F F <--- 28 560-579 60 57 60 97 95 97 F F F <--- 29 580-599 52 52 52 88 88 88 F F F Max gen: 60, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 97 97 97 F F F 1 020-039 60 60 60 97 97 97 F F F 2 040-059 60 60 60 97 97 97 F F F 3 060-079 57 57 57 95 95 95 F F F 4 080-099 61 61 61 99 99 99 F F F 5 100-119 61 61 61 99 99 99 F F F 6 120-139 58 58 58 96 96 96 F F F 7 140-159 60 60 60 97 97 97 F F F 8 160-179 60 60 60 97 97 97 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 57 57 57 95 95 95 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 58 58 58 96 96 96 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 60 60 60 98 98 98 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 61 61 61 99 99 99 F F F 24 480-499 56 56 56 93 93 93 F F F 25 500-519 56 56 56 93 93 93 F F F 26 520-539 60 60 60 97 97 97 F F F 27 540-559 60 60 60 97 97 97 F F F 28 560-579 60 60 60 97 97 97 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 61, Max flush: 99 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.187445706Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.195622568Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.19589991Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.195925818Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.195937027Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.195948986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Crucible 07b24b8b-bff9-4201-8863-2725cc04632a has session id: fa0bfdef-8cc1-4f16-82e6-d789d2348803","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196299963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Upstairs opts: Upstairs UUID: 07b24b8b-bff9-4201-8863-2725cc04632a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196328112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Crucible stats registered with UUID: 07b24b8b-bff9-4201-8863-2725cc04632a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196339081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:33.19634832Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.19649681Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196512629Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196522749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.19665463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196673509Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"07b24b8b-bff9-4201-8863-2725cc04632a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196699847Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196709086Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"0","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196718806Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"1","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196728495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"2","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196740824Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196750724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"ds_connection connected from Ok(127.0.0.1:64956)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196826629Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"ds_connection connected from Ok(127.0.0.1:42656)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196873166Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196894504Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"ds_connection connected from Ok(127.0.0.1:48432)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.196905764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.197585769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"0","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.197605178Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"1","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.197640605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"2","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.197896528Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.197916587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[0]R flush_numbers[0..12]: [97, 97, 97, 95, 99, 99, 96, 97, 97, 97, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198001941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[0]R generation[0..12]: [60, 60, 60, 57, 61, 61, 58, 60, 60, 60, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198013221Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.19802428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[1]R flush_numbers[0..12]: [97, 97, 97, 95, 99, 99, 96, 97, 97, 97, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198038249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[1]R generation[0..12]: [60, 60, 60, 57, 61, 61, 58, 60, 60, 60, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198048848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198059998Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[2]R flush_numbers[0..12]: [97, 97, 97, 95, 99, 99, 96, 97, 97, 97, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198070487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[2]R generation[0..12]: [60, 60, 60, 57, 61, 61, 58, 60, 60, 60, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198080866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198091905Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198101295Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198111054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198120254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198129503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"":"downstairs","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198139282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198148132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"0","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198157761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"1","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198167301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"client":"2","":"downstairs","session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.19817702Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198186289Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"07b24b8b-bff9-4201-8863-2725cc04632a is now active with session: fa0bfdef-8cc1-4f16-82e6-d789d2348803","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198198049Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198206848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"session_id":"fa0bfdef-8cc1-4f16-82e6-d789d2348803"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198215947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Activated sub_volume 07b24b8b-bff9-4201-8863-2725cc04632a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198223867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198232006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.198531827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"task":"crutest"} 01/30 Write block 495 len 20480 data: 6 6 7 7 9 02/30 Write block 494 len 20480 data: 8 7 7 8 8 03/30 Read block 242 len 4096 04/30 Read block 500 len 32768 05/30 Read block 518 len 32768 06/30 Read block 146 len 36864 07/30 Flush 08/30 Write block 503 len 28672 data: 3 3 3 3 2 2 3 09/30 Read block 564 len 4096 10/30 Read block 334 len 40960 11/30 Write block 421 len 24576 data: 5 7 5 6 6 6 12/30 Read block 342 len 20480 13/30 Read block 168 len 32768 14/30 Read block 547 len 20480 15/30 Flush 16/30 Write block 287 len 20480 data: 5 7 6 7 7 17/30 Write block 594 len 16384 data: 6 5 6 6 18/30 Read block 582 len 12288 19/30 Read block 244 len 36864 20/30 Write block 477 len 28672 data: 5 6 6 6 7 7 6 21/30 Read block 289 len 8192 22/30 Read block 196 len 20480 23/30 Read block 260 len 36864 24/30 Read block 74 len 12288 25/30 Read block 238 len 36864 26/30 Write block 544 len 28672 data: 5 6 6 9 8 9 9 27/30 Write block 427 len 40960 data: 6 4 4 5 5 5 3 4 5 6 28/30 Read block 592 len 4096 29/30 Write block 21 len 12288 data: 7 6 6 30/30 Write block 174 len 8192 data: 6 5 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:33.207339537Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:33.207519305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1384,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 97 97 97 F F F 1 020-039 60 60 60 97 97 97 T F T <--- 2 040-059 60 60 60 97 97 97 F F F 3 060-079 57 57 57 95 95 95 F F F 4 080-099 61 61 61 99 99 99 F F F 5 100-119 61 61 61 99 99 99 F F F 6 120-139 58 58 58 96 96 96 F F F 7 140-159 60 60 60 97 97 97 F F F 8 160-179 60 60 60 97 97 97 T F T <--- 9 180-199 60 60 60 97 97 97 F F F 10 200-219 57 57 57 95 95 95 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 52 52 52 88 88 88 T F T <--- 15 300-319 61 61 61 99 99 99 F F F 16 320-339 58 58 58 96 96 96 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 62 60 62 101 98 101 T F T <--- 22 440-459 58 58 58 96 96 96 F F F 23 460-479 61 61 61 99 99 99 T F T <--- 24 480-499 62 56 62 100 93 100 T T T <--- 25 500-519 62 56 62 101 93 101 F F F <--- 26 520-539 60 60 60 97 97 97 F F F 27 540-559 60 60 60 97 97 97 T F T <--- 28 560-579 60 60 60 97 97 97 F F F 29 580-599 52 52 52 88 88 88 T F T <--- Max gen: 62, Max flush: 101 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 97 97 97 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 60 60 60 97 97 97 F F F 3 060-079 57 57 57 95 95 95 F F F 4 080-099 61 61 61 99 99 99 F F F 5 100-119 61 61 61 99 99 99 F F F 6 120-139 58 58 58 96 96 96 F F F 7 140-159 60 60 60 97 97 97 F F F 8 160-179 63 63 63 102 102 102 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 57 57 57 95 95 95 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 61 61 61 99 99 99 F F F 16 320-339 58 58 58 96 96 96 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 63 63 63 102 102 102 F F F 22 440-459 58 58 58 96 96 96 F F F 23 460-479 63 63 63 102 102 102 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 62 62 62 101 101 101 F F F 26 520-539 60 60 60 97 97 97 F F F 27 540-559 63 63 63 102 102 102 F F F 28 560-579 60 60 60 97 97 97 F F F 29 580-599 63 63 63 102 102 102 F F F Max gen: 63, Max flush: 102 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.454888734Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463096144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463391725Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463426043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463435322Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463447311Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Crucible 83d85a57-cc48-4425-a4fc-1e2b4c2a4301 has session id: e0c7415e-7ce9-486d-b30b-813508686d1b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463737272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Upstairs opts: Upstairs UUID: 83d85a57-cc48-4425-a4fc-1e2b4c2a4301, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463762771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Crucible stats registered with UUID: 83d85a57-cc48-4425-a4fc-1e2b4c2a4301","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.46377288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:37.463785089Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.46392525Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463953198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.463964357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464109228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464124227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"83d85a57-cc48-4425-a4fc-1e2b4c2a4301 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464160844Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464170324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"0","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464179903Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"1","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464189552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"2","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464199282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464209251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464219171Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36726)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464236699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61480)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464328573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53486)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464403828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464893496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"0","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.464978091Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"2","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465017988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"1","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465268192Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465277871Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[0]R flush_numbers[0..12]: [97, 102, 97, 95, 99, 99, 96, 97, 102, 97, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.46529526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[0]R generation[0..12]: [60, 63, 60, 57, 61, 61, 58, 60, 63, 60, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465306079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465317378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[1]R flush_numbers[0..12]: [97, 102, 97, 95, 99, 99, 96, 97, 102, 97, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465327898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[1]R generation[0..12]: [60, 63, 60, 57, 61, 61, 58, 60, 63, 60, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465338247Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465349116Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[2]R flush_numbers[0..12]: [97, 102, 97, 95, 99, 99, 96, 97, 102, 97, 95, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465359566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[2]R generation[0..12]: [60, 63, 60, 57, 61, 61, 58, 60, 63, 60, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465370125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465380954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465393063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465402873Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465412222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465421641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"":"downstairs","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465431371Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.46544008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"0","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.46544965Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"1","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465459259Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"client":"2","":"downstairs","session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465469078Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465478278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"83d85a57-cc48-4425-a4fc-1e2b4c2a4301 is now active with session: e0c7415e-7ce9-486d-b30b-813508686d1b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465488267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465496907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"session_id":"e0c7415e-7ce9-486d-b30b-813508686d1b"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465505986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Activated sub_volume 83d85a57-cc48-4425-a4fc-1e2b4c2a4301","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465514395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465522685Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.465837154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"task":"crutest"} 01/30 Flush 02/30 Write block 214 len 12288 data: 12 12 10 03/30 Write block 473 len 12288 data: 5 4 5 04/30 Write block 471 len 36864 data: 6 7 6 5 6 6 6 7 7 05/30 Write block 576 len 20480 data: 6 6 5 4 4 06/30 Write block 151 len 28672 data: 10 10 8 8 5 7 8 07/30 Read block 169 len 32768 08/30 Write block 63 len 4096 data: 5 09/30 Write block 67 len 40960 data: 3 3 7 6 5 6 7 7 7 7 10/30 Flush 11/30 Write block 92 len 16384 data: 4 4 4 4 12/30 Write block 322 len 40960 data: 8 8 8 6 7 7 8 8 7 6 13/30 Write block 447 len 20480 data: 6 7 7 6 6 14/30 Flush 15/30 Write block 538 len 32768 data: 4 4 5 5 5 6 6 7 16/30 Read block 543 len 32768 17/30 Read block 374 len 36864 18/30 Write block 111 len 40960 data: 5 5 4 3 3 4 5 6 8 8 19/30 Write block 42 len 36864 data: 4 4 5 4 4 5 5 4 6 20/30 Write block 319 len 24576 data: 8 7 8 9 9 9 21/30 Read block 583 len 24576 22/30 Read block 590 len 36864 23/30 Read block 342 len 4096 24/30 Write block 552 len 16384 data: 6 6 6 8 25/30 Write block 303 len 8192 data: 6 7 26/30 Read block 11 len 40960 27/30 Write block 457 len 24576 data: 4 4 6 6 5 6 28/30 Write block 5 len 12288 data: 7 7 7 29/30 Write block 185 len 12288 data: 6 5 6 30/30 Write block 442 len 20480 data: 5 5 4 5 5 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 6 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 30 4 38 Done 26 0 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:37.473436284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:37.473594174Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1392,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 97 97 97 T F T <--- 1 020-039 63 63 63 102 102 102 F F F 2 040-059 60 60 60 97 97 97 T F T <--- 3 060-079 64 57 64 104 95 104 F F F <--- 4 080-099 64 61 64 105 99 105 F F F <--- 5 100-119 61 61 61 99 99 99 T F T <--- 6 120-139 58 58 58 96 96 96 T F T <--- 7 140-159 64 60 64 104 97 104 F F F <--- 8 160-179 63 63 63 102 102 102 F F F 9 180-199 60 60 60 97 97 97 T F T <--- 10 200-219 64 57 64 104 95 104 F F F <--- 11 220-239 58 58 58 96 96 96 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 61 61 61 99 99 99 T F T <--- 16 320-339 64 58 64 105 96 105 T F T <--- 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 63 63 63 102 102 102 F F F 22 440-459 64 58 64 105 96 105 T F T <--- 23 460-479 64 63 64 104 102 104 T F T <--- 24 480-499 63 63 63 102 102 102 F F F 25 500-519 62 62 62 101 101 101 F F F 26 520-539 60 60 60 97 97 97 T F T <--- 27 540-559 63 63 63 102 102 102 T F T <--- 28 560-579 64 60 64 104 97 104 F F F <--- 29 580-599 64 63 64 104 102 104 F F F <--- Max gen: 64, Max flush: 105 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 65 65 65 106 106 106 F F F 3 060-079 64 64 64 104 104 104 F F F 4 080-099 64 64 64 105 105 105 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 65 65 65 106 106 106 F F F 7 140-159 64 64 64 104 104 104 F F F 8 160-179 63 63 63 102 102 102 F F F 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 65 65 65 106 106 106 F F F 16 320-339 65 65 65 106 106 106 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 54 54 90 90 90 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 63 63 63 102 102 102 F F F 22 440-459 65 65 65 106 106 106 F F F 23 460-479 65 65 65 106 106 106 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 62 62 62 101 101 101 F F F 26 520-539 65 65 65 106 106 106 F F F 27 540-559 65 65 65 106 106 106 F F F 28 560-579 64 64 64 104 104 104 F F F 29 580-599 64 64 64 104 104 104 F F F Max gen: 65, Max flush: 106 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.738050139Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:41.745368098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.745434613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.752729924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753002806Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753052122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753061092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753072891Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Crucible 9f0ea155-ea6e-40be-83f2-70863acf027f has session id: 5492310e-aa41-4afd-9ce7-02ba6a660d03","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753382571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Upstairs opts: Upstairs UUID: 9f0ea155-ea6e-40be-83f2-70863acf027f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753406419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Crucible stats registered with UUID: 9f0ea155-ea6e-40be-83f2-70863acf027f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753416388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:41.753425358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753582907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753612886Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753626545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753761266Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753777035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"9f0ea155-ea6e-40be-83f2-70863acf027f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753812282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753821692Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"0","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753831431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"1","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.75384124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"2","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.75385097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753875628Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753901187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"ds_connection connected from Ok(127.0.0.1:56524)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.753973732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"ds_connection connected from Ok(127.0.0.1:46821)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.754081325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"ds_connection connected from Ok(127.0.0.1:59794)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.754093414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.754730812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"1","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.754745351Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"2","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.75475632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"0","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755109707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755119196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[0]R flush_numbers[0..12]: [106, 102, 106, 104, 105, 106, 106, 104, 102, 106, 104, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755159084Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[0]R generation[0..12]: [65, 63, 65, 64, 64, 65, 65, 64, 63, 65, 64, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755185052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755197131Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[1]R flush_numbers[0..12]: [106, 102, 106, 104, 105, 106, 106, 104, 102, 106, 104, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755208041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[1]R generation[0..12]: [65, 63, 65, 64, 64, 65, 65, 64, 63, 65, 64, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.7552186Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755229899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[2]R flush_numbers[0..12]: [106, 102, 106, 104, 105, 106, 106, 104, 102, 106, 104, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755240678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[2]R generation[0..12]: [65, 63, 65, 64, 64, 65, 65, 64, 63, 65, 64, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755251138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755262237Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755271806Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755281716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755291305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755303324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"":"downstairs","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755313124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755321933Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"0","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755331612Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"1","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755341352Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"client":"2","":"downstairs","session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755351351Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755360491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"9f0ea155-ea6e-40be-83f2-70863acf027f is now active with session: 5492310e-aa41-4afd-9ce7-02ba6a660d03","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.75537039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755379299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"5492310e-aa41-4afd-9ce7-02ba6a660d03"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755394048Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Activated sub_volume 9f0ea155-ea6e-40be-83f2-70863acf027f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755401978Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755410127Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.755646132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} 01/30 Write block 318 len 4096 data: 6 02/30 Read block 448 len 24576 03/30 Flush 04/30 Write block 68 len 32768 data: 4 8 7 6 7 8 8 8 05/30 Write block 116 len 12288 data: 5 6 7 06/30 Read block 320 len 32768 07/30 Read block 328 len 8192 08/30 Write block 134 len 40960 data: 7 8 6 7 6 5 4 7 10 9 09/30 Read block 582 len 28672 10/30 Read block 286 len 36864 11/30 Write block 501 len 32768 data: 6 4 4 4 4 4 3 3 12/30 Read block 430 len 32768 13/30 Write block 50 len 16384 data: 7 7 6 5 14/30 Read block 313 len 40960 15/30 Write block 563 len 20480 data: 7 6 5 7 8 16/30 Write block 63 len 40960 data: 6 4 5 3 4 5 9 8 7 8 17/30 Flush 18/30 Write block 464 len 40960 data: 6 7 5 5 4 5 5 7 8 7 19/30 Read block 382 len 8192 20/30 Read block 550 len 12288 21/30 Flush 22/30 Write block 382 len 8192 data: 2 2 23/30 Write block 69 len 8192 data: 10 9 24/30 Flush 25/30 Write block 224 len 24576 data: 3 3 3 5 5 5 26/30 Write block 434 len 24576 data: 5 6 7 7 8 7 27/30 Read block 9 len 12288 28/30 Read block 342 len 16384 29/30 Write block 255 len 16384 data: 4 4 3 3 30/30 Write block 71 len 24576 data: 8 9 9 9 9 8 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:41.764827908Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:41.764990907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1400,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 65 66 66 106 108 108 F F F <--- 3 060-079 64 66 66 104 110 110 F T T <--- 4 080-099 64 64 64 105 105 105 F F F 5 100-119 65 66 66 106 108 108 F F F <--- 6 120-139 65 66 66 106 108 108 F F F <--- 7 140-159 64 66 66 104 108 108 F F F <--- 8 160-179 63 63 63 102 102 102 F F F 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 58 58 58 96 96 96 F T T <--- 12 240-259 61 61 61 99 99 99 F T T <--- 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 65 66 66 106 107 107 F F F <--- 16 320-339 65 65 65 106 106 106 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 54 66 66 90 110 110 F F F <--- 20 400-419 53 53 53 89 89 89 F F F 21 420-439 63 63 63 102 102 102 F T T <--- 22 440-459 65 65 65 106 106 106 F F F 23 460-479 65 66 66 106 109 109 F F F <--- 24 480-499 63 63 63 102 102 102 F F F 25 500-519 62 66 66 101 108 108 F F F <--- 26 520-539 65 65 65 106 106 106 F F F 27 540-559 65 65 65 106 106 106 F F F 28 560-579 64 66 66 104 108 108 F F F <--- 29 580-599 64 64 64 104 104 104 F F F Max gen: 66, Max flush: 110 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 66 66 66 108 108 108 F F F 3 060-079 67 67 67 111 111 111 F F F 4 080-099 64 64 64 105 105 105 F F F 5 100-119 66 66 66 108 108 108 F F F 6 120-139 66 66 66 108 108 108 F F F 7 140-159 66 66 66 108 108 108 F F F 8 160-179 63 63 63 102 102 102 F F F 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 67 67 67 111 111 111 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 66 66 66 107 107 107 F F F 16 320-339 65 65 65 106 106 106 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 66 66 66 110 110 110 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 67 67 67 111 111 111 F F F 22 440-459 65 65 65 106 106 106 F F F 23 460-479 66 66 66 109 109 109 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 66 66 66 108 108 108 F F F 26 520-539 65 65 65 106 106 106 F F F 27 540-559 65 65 65 106 106 106 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 64 64 64 104 104 104 F F F Max gen: 67, Max flush: 111 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.044089399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.052465258Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.05274784Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.052772308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.052783758Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.052795597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Crucible 01fb7178-91e5-4b35-b7ab-83144412de00 has session id: 619ad7e6-390b-4652-8a5f-a6c9442f10cc","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053101757Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Upstairs opts: Upstairs UUID: 01fb7178-91e5-4b35-b7ab-83144412de00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053165102Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Crucible stats registered with UUID: 01fb7178-91e5-4b35-b7ab-83144412de00","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053189421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:46.0531995Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053328742Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053344111Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.05335429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053497161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053520959Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"01fb7178-91e5-4b35-b7ab-83144412de00 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053534758Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053543678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"0","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053553407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"1","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053565386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"2","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053575075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053584925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053594784Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35088)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.05366284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:46335)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053691968Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:65197)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.053703707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054329436Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"1","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054375533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"2","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054392422Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"0","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054653735Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054688742Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[0]R flush_numbers[0..12]: [106, 102, 108, 111, 105, 108, 108, 108, 102, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054769747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[0]R generation[0..12]: [65, 63, 66, 67, 64, 66, 66, 66, 63, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054781156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054792555Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[1]R flush_numbers[0..12]: [106, 102, 108, 111, 105, 108, 108, 108, 102, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054806864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[1]R generation[0..12]: [65, 63, 66, 67, 64, 66, 66, 66, 63, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054817324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054828373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[2]R flush_numbers[0..12]: [106, 102, 108, 111, 105, 108, 108, 108, 102, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054839152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[2]R generation[0..12]: [65, 63, 66, 67, 64, 66, 66, 66, 63, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054849532Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054860301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.05486964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.05487964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054888989Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054898438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"":"downstairs","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054908168Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054917047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"0","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054926697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"1","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054936366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"client":"2","":"downstairs","session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054946285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054957025Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"01fb7178-91e5-4b35-b7ab-83144412de00 is now active with session: 619ad7e6-390b-4652-8a5f-a6c9442f10cc","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054966884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054975613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"session_id":"619ad7e6-390b-4652-8a5f-a6c9442f10cc"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054984693Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Activated sub_volume 01fb7178-91e5-4b35-b7ab-83144412de00","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.054992982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.055001012Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.055302652Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"task":"crutest"} 01/30 Write block 53 len 20480 data: 6 5 4 5 5 02/30 Write block 319 len 8192 data: 9 8 03/30 Write block 510 len 12288 data: 3 3 5 04/30 Read block 106 len 4096 05/30 Flush 06/30 Write block 332 len 20480 data: 7 7 4 5 7 07/30 Read block 497 len 36864 08/30 Read block 500 len 24576 09/30 Read block 123 len 16384 10/30 Read block 213 len 36864 11/30 Flush 12/30 Write block 346 len 36864 data: 8 8 6 6 6 6 6 5 5 13/30 Flush 14/30 Write block 586 len 24576 data: 6 4 5 5 6 6 15/30 Write block 440 len 12288 data: 6 6 6 16/30 Write block 174 len 16384 data: 7 6 5 7 17/30 Write block 123 len 24576 data: 8 8 6 5 6 6 18/30 Flush 19/30 Write block 544 len 8192 data: 7 8 20/30 Read block 184 len 12288 21/30 Write block 89 len 32768 data: 4 5 5 5 5 5 5 5 22/30 Read block 185 len 16384 23/30 Write block 159 len 28672 data: 9 7 7 6 5 4 4 24/30 Read block 275 len 28672 25/30 Write block 53 len 8192 data: 7 6 26/30 Read block 332 len 8192 27/30 Write block 80 len 20480 data: 3 3 2 2 3 28/30 Write block 441 len 40960 data: 7 7 6 5 6 6 7 8 8 7 29/30 Read block 273 len 36864 30/30 Read block 469 len 16384 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 3 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 28 29 Done 29 30 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:46.06384845Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:46.064036617Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1408,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 68 68 66 112 112 108 T T T <--- 3 060-079 67 67 67 111 111 111 F F F 4 080-099 64 64 64 105 105 105 T T F <--- 5 100-119 66 66 66 108 108 108 F F F 6 120-139 68 68 66 115 115 108 F F F <--- 7 140-159 66 66 66 108 108 108 T T F <--- 8 160-179 68 68 63 115 115 102 T T F <--- 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 67 67 67 111 111 111 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 68 68 66 112 112 107 F F T <--- 16 320-339 68 68 65 113 113 106 F F T <--- 17 340-359 68 68 54 114 114 90 F F F <--- 18 360-379 58 58 58 96 96 96 F F F 19 380-399 66 66 66 110 110 110 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 67 67 67 111 111 111 F F F 22 440-459 68 68 65 115 115 106 T T F <--- 23 460-479 66 66 66 109 109 109 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 68 68 66 112 112 108 F F F <--- 26 520-539 65 65 65 106 106 106 F F F 27 540-559 65 65 65 106 106 106 T T F <--- 28 560-579 66 66 66 108 108 108 F F F 29 580-599 68 68 64 115 115 104 F F F <--- Max gen: 68, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 69 69 69 116 116 116 F F F 3 060-079 67 67 67 111 111 111 F F F 4 080-099 69 69 69 116 116 116 F F F 5 100-119 66 66 66 108 108 108 F F F 6 120-139 68 68 68 115 115 115 F F F 7 140-159 69 69 69 116 116 116 F F F 8 160-179 69 69 69 116 116 116 F F F 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 67 67 67 111 111 111 F F F 13 260-279 58 58 58 96 96 96 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 68 68 68 112 112 112 F F F 16 320-339 68 68 68 113 113 113 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 66 66 66 110 110 110 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 67 67 67 111 111 111 F F F 22 440-459 69 69 69 116 116 116 F F F 23 460-479 66 66 66 109 109 109 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 68 68 68 112 112 112 F F F 26 520-539 65 65 65 106 106 106 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 68 68 68 115 115 115 F F F Max gen: 69, Max flush: 116 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.315520175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32380234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324079462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32410884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324117679Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324129639Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Crucible 71655874-c433-4ce0-a469-b6c448c9ff10 has session id: bc6cd7d7-4c28-46a8-a029-86c371f23595","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324469416Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Upstairs opts: Upstairs UUID: 71655874-c433-4ce0-a469-b6c448c9ff10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324498914Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Crucible stats registered with UUID: 71655874-c433-4ce0-a469-b6c448c9ff10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324513063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:50.324522403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324651464Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324680512Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324691412Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324835812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32487022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"71655874-c433-4ce0-a469-b6c448c9ff10 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324880049Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324889069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"0","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324898878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"1","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324908737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"2","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324918487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324928506Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.324938475Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"ds_connection connected from Ok(127.0.0.1:57005)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32501371Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"ds_connection connected from Ok(127.0.0.1:59711)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.325156561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"ds_connection connected from Ok(127.0.0.1:34175)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.325185249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32562134Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"0","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.325648479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"1","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.325747872Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"2","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.325973867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.325983567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[0]R flush_numbers[0..12]: [106, 102, 116, 111, 116, 108, 115, 116, 116, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326018074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[0]R generation[0..12]: [65, 63, 69, 67, 69, 66, 68, 69, 69, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326029234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326040283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[1]R flush_numbers[0..12]: [106, 102, 116, 111, 116, 108, 115, 116, 116, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326051022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[1]R generation[0..12]: [65, 63, 69, 67, 69, 66, 68, 69, 69, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326061421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326072351Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[2]R flush_numbers[0..12]: [106, 102, 116, 111, 116, 108, 115, 116, 116, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32608298Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[2]R generation[0..12]: [65, 63, 69, 67, 69, 66, 68, 69, 69, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326093319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326107248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326116728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326126427Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326135607Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326144946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"":"downstairs","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326154675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326163505Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"0","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326173114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"1","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326182783Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"client":"2","":"downstairs","session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326192683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326202052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"71655874-c433-4ce0-a469-b6c448c9ff10 is now active with session: bc6cd7d7-4c28-46a8-a029-86c371f23595","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326212062Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326220931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 bc6cd7d7-4c28-46a8-a029-86c371f23595"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.32623508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Activated sub_volume 71655874-c433-4ce0-a469-b6c448c9ff10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326243139Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326251379Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.326483654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"task":"crutest"} 01/30 Flush 02/30 Write block 444 len 28672 data: 6 7 7 8 9 9 8 03/30 Write block 102 len 8192 data: 10 10 04/30 Read block 532 len 20480 05/30 Write block 260 len 8192 data: 3 4 06/30 Flush 07/30 Write block 48 len 8192 data: 6 5 08/30 Read block 219 len 4096 09/30 Read block 200 len 24576 10/30 Write block 256 len 4096 data: 5 11/30 Read block 469 len 28672 12/30 Read block 59 len 28672 13/30 Write block 268 len 36864 data: 6 5 4 4 5 5 6 5 5 14/30 Read block 469 len 28672 15/30 Read block 94 len 28672 16/30 Read block 245 len 8192 17/30 Read block 329 len 40960 18/30 Read block 222 len 32768 19/30 Flush 20/30 Write block 95 len 24576 data: 6 6 6 8 9 10 21/30 Write block 526 len 8192 data: 4 4 22/30 Write block 303 len 8192 data: 7 8 23/30 Write block 338 len 20480 data: 8 7 8 9 8 24/30 Write block 384 len 32768 data: 2 3 4 5 6 6 7 8 25/30 Read block 159 len 24576 26/30 Read block 179 len 12288 27/30 Write block 470 len 32768 data: 6 8 9 8 6 7 7 7 28/30 Read block 584 len 12288 29/30 Read block 413 len 28672 30/30 Read block 32 len 16384 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 7 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:50.335017262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:50.3352059Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1416,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 70 70 69 119 119 116 F F F <--- 3 060-079 67 67 67 111 111 111 F F F 4 080-099 69 69 69 116 116 116 T T F <--- 5 100-119 70 70 66 118 118 108 T T F <--- 6 120-139 68 68 68 115 115 115 F F F 7 140-159 69 69 69 116 116 116 F F F 8 160-179 69 69 69 116 116 116 F F F 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 70 70 67 119 119 111 F F F <--- 13 260-279 70 70 58 119 119 96 F F F <--- 14 280-299 63 63 63 102 102 102 F F F 15 300-319 68 68 68 112 112 112 T T F <--- 16 320-339 68 68 68 113 113 113 T T F <--- 17 340-359 68 68 68 114 114 114 T T F <--- 18 360-379 58 58 58 96 96 96 F F F 19 380-399 66 66 66 110 110 110 T T F <--- 20 400-419 53 53 53 89 89 89 F F F 21 420-439 67 67 67 111 111 111 F F F 22 440-459 70 70 69 118 118 116 F F F <--- 23 460-479 66 66 66 109 109 109 T T F <--- 24 480-499 63 63 63 102 102 102 F F F 25 500-519 68 68 68 112 112 112 F F F 26 520-539 65 65 65 106 106 106 T T F <--- 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 68 68 68 115 115 115 F F F Max gen: 70, Max flush: 119 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 70 70 70 119 119 119 F F F 3 060-079 67 67 67 111 111 111 F F F 4 080-099 71 71 71 120 120 120 F F F 5 100-119 71 71 71 120 120 120 F F F 6 120-139 68 68 68 115 115 115 F F F 7 140-159 69 69 69 116 116 116 F F F 8 160-179 69 69 69 116 116 116 F F F 9 180-199 65 65 65 106 106 106 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 70 70 70 119 119 119 F F F 13 260-279 70 70 70 119 119 119 F F F 14 280-299 63 63 63 102 102 102 F F F 15 300-319 71 71 71 120 120 120 F F F 16 320-339 71 71 71 120 120 120 F F F 17 340-359 71 71 71 120 120 120 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 71 71 71 120 120 120 F F F 20 400-419 53 53 53 89 89 89 F F F 21 420-439 67 67 67 111 111 111 F F F 22 440-459 70 70 70 118 118 118 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 68 68 68 112 112 112 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 68 68 68 115 115 115 F F F Max gen: 71, Max flush: 120 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.585518695Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.59365283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.593926652Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.59395678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.59396518Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.593976659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Crucible 6e230fdf-5185-42a8-8917-7b3454623a89 has session id: fcf6b702-81b2-46e4-9151-1b8a6f481f9b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594321436Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Upstairs opts: Upstairs UUID: 6e230fdf-5185-42a8-8917-7b3454623a89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594350275Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Crucible stats registered with UUID: 6e230fdf-5185-42a8-8917-7b3454623a89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594361354Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:54.594370663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594505414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594535272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594546182Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594696562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594712401Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"6e230fdf-5185-42a8-8917-7b3454623a89 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594743819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594753198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"0","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594762877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"1","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594772637Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"2","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594782376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594792235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594802105Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65359)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594845752Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33842)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594856791Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55499)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.594867081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595558095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"1","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595576394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"0","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595587453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"2","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595883294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595909282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[0]R flush_numbers[0..12]: [106, 102, 119, 111, 120, 120, 115, 116, 116, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595956599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[0]R generation[0..12]: [65, 63, 70, 67, 71, 71, 68, 69, 69, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595968268Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595979457Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[1]R flush_numbers[0..12]: [106, 102, 119, 111, 120, 120, 115, 116, 116, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.595990197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[1]R generation[0..12]: [65, 63, 70, 67, 71, 71, 68, 69, 69, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596014455Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596025384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[2]R flush_numbers[0..12]: [106, 102, 119, 111, 120, 120, 115, 116, 116, 106, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596035944Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[2]R generation[0..12]: [65, 63, 70, 67, 71, 71, 68, 69, 69, 65, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596046183Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596057172Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596066622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596076281Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.59608563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.59609526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"":"downstairs","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596106519Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596115408Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"0","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596125468Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"1","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596135297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"client":"2","":"downstairs","session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596145386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596154606Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"6e230fdf-5185-42a8-8917-7b3454623a89 is now active with session: fcf6b702-81b2-46e4-9151-1b8a6f481f9b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596164665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596173375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"session_id":"fcf6b702-81b2-46e4-9151-1b8a6f481f9b"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596182344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Activated sub_volume 6e230fdf-5185-42a8-8917-7b3454623a89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596190254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596198333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.596448117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"task":"crutest"} 01/30 Write block 70 len 28672 data: 10 9 10 10 10 10 9 02/30 Read block 161 len 4096 03/30 Write block 4 len 40960 data: 5 8 8 8 9 5 6 5 6 4 04/30 Read block 124 len 24576 05/30 Write block 279 len 24576 data: 3 2 3 4 4 5 06/30 Read block 8 len 32768 07/30 Read block 470 len 20480 08/30 Write block 6 len 24576 data: 9 9 10 6 7 6 09/30 Read block 390 len 12288 10/30 Flush 11/30 Flush 12/30 Write block 505 len 8192 data: 5 5 13/30 Read block 326 len 4096 14/30 Write block 119 len 36864 data: 9 9 8 8 9 9 7 6 7 15/30 Write block 418 len 36864 data: 4 4 4 6 8 6 7 7 7 16/30 Write block 185 len 40960 data: 7 6 7 7 6 6 3 4 5 5 17/30 Read block 202 len 8192 18/30 Read block 279 len 24576 19/30 Read block 506 len 20480 20/30 Read block 193 len 28672 21/30 Read block 153 len 4096 22/30 Write block 125 len 40960 data: 8 7 8 7 6 5 6 6 6 8 23/30 Write block 261 len 4096 data: 5 24/30 Read block 574 len 20480 25/30 Read block 114 len 24576 26/30 Write block 592 len 16384 data: 6 5 7 6 27/30 Read block 9 len 4096 28/30 Write block 357 len 40960 data: 7 5 7 6 6 8 9 10 8 10 29/30 Write block 131 len 16384 data: 7 7 7 9 30/30 Read block 494 len 16384 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 21 22 Done 30 29 9 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:54.6053614Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:54.60552501Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1424,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 65 121 121 106 F F T <--- 1 020-039 63 63 63 102 102 102 F F F 2 040-059 70 70 70 119 119 119 F F F 3 060-079 72 72 67 121 121 111 F F T <--- 4 080-099 71 71 71 120 120 120 F F F 5 100-119 71 71 71 120 120 120 T T F <--- 6 120-139 68 68 68 115 115 115 T T F <--- 7 140-159 69 69 69 116 116 116 F F F 8 160-179 69 69 69 116 116 116 F F F 9 180-199 65 65 65 106 106 106 T T F <--- 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 70 70 70 119 119 119 F F F 13 260-279 72 72 70 121 121 119 T T T <--- 14 280-299 72 72 63 121 121 102 F F T <--- 15 300-319 71 71 71 120 120 120 F F F 16 320-339 71 71 71 120 120 120 F F F 17 340-359 71 71 71 120 120 120 T T F <--- 18 360-379 58 58 58 96 96 96 T T F <--- 19 380-399 71 71 71 120 120 120 F F F 20 400-419 53 53 53 89 89 89 T T F <--- 21 420-439 67 67 67 111 111 111 T T F <--- 22 440-459 70 70 70 118 118 118 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 68 68 68 112 112 112 T T F <--- 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 68 68 68 115 115 115 T T F <--- Max gen: 72, Max flush: 121 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 121 121 121 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 70 70 70 119 119 119 F F F 3 060-079 72 72 72 121 121 121 F F F 4 080-099 71 71 71 120 120 120 F F F 5 100-119 73 73 73 122 122 122 F F F 6 120-139 73 73 73 122 122 122 F F F 7 140-159 69 69 69 116 116 116 F F F 8 160-179 69 69 69 116 116 116 F F F 9 180-199 73 73 73 122 122 122 F F F 10 200-219 64 64 64 104 104 104 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 70 70 70 119 119 119 F F F 13 260-279 73 73 73 122 122 122 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 71 71 71 120 120 120 F F F 16 320-339 71 71 71 120 120 120 F F F 17 340-359 73 73 73 122 122 122 F F F 18 360-379 73 73 73 122 122 122 F F F 19 380-399 71 71 71 120 120 120 F F F 20 400-419 73 73 73 122 122 122 F F F 21 420-439 73 73 73 122 122 122 F F F 22 440-459 70 70 70 118 118 118 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 73 73 73 122 122 122 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 73 73 73 122 122 122 F F F Max gen: 73, Max flush: 122 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.860267253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:58.867529295Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.867578182Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.874543114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.874800087Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.874826175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.874834675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.874846384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Crucible d4ee6cb1-b02b-4ceb-b8a1-9048abbdad14 has session id: 5e60792b-bf0a-47f5-bed4-e247496a2eaf","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.87520605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Upstairs opts: Upstairs UUID: d4ee6cb1-b02b-4ceb-b8a1-9048abbdad14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875226469Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Crucible stats registered with UUID: d4ee6cb1-b02b-4ceb-b8a1-9048abbdad14","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875236738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:58.875245927Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875383158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875397977Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875408127Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875552397Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875568046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"d4ee6cb1-b02b-4ceb-b8a1-9048abbdad14 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875601694Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875611153Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"0","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875623023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"1","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875632942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"2","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875642381Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875652491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.87566251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:62621)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875732955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:47292)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.87581354Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:59553)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.875825359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876360964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"1","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876381333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"0","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876392372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"2","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876660374Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876685453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[0]R flush_numbers[0..12]: [121, 102, 119, 121, 120, 122, 122, 116, 116, 122, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.87673407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[0]R generation[0..12]: [72, 63, 70, 72, 71, 73, 73, 69, 69, 73, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876745219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876759118Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[1]R flush_numbers[0..12]: [121, 102, 119, 121, 120, 122, 122, 116, 116, 122, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876769967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[1]R generation[0..12]: [72, 63, 70, 72, 71, 73, 73, 69, 69, 73, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876780277Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876791146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[2]R flush_numbers[0..12]: [121, 102, 119, 121, 120, 122, 122, 116, 116, 122, 104, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876801805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[2]R generation[0..12]: [72, 63, 70, 72, 71, 73, 73, 69, 69, 73, 64, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876812124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876823114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876832423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876842262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876851942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876861301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"":"downstairs","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876871421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.8768805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"0","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876890529Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"1","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876900309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"client":"2","":"downstairs","session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876911708Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876920917Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"d4ee6cb1-b02b-4ceb-b8a1-9048abbdad14 is now active with session: 5e60792b-bf0a-47f5-bed4-e247496a2eaf","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876931207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876940066Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"session_id":"5e60792b-bf0a-47f5-bed4-e247496a2eaf"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876949255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Activated sub_volume d4ee6cb1-b02b-4ceb-b8a1-9048abbdad14","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876957395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.876965584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.877277364Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} 01/30 Write block 208 len 36864 data: 10 11 11 14 14 13 13 13 11 02/30 Flush 03/30 Write block 273 len 8192 data: 6 7 04/30 Read block 236 len 20480 05/30 Read block 247 len 12288 06/30 Read block 169 len 20480 07/30 Flush 08/30 Write block 393 len 8192 data: 8 7 09/30 Read block 183 len 20480 10/30 Write block 59 len 24576 data: 4 4 4 4 7 5 11/30 Read block 14 len 24576 12/30 Read block 103 len 40960 13/30 Flush 14/30 Write block 390 len 36864 data: 8 9 9 9 8 7 5 6 6 15/30 Read block 564 len 32768 16/30 Write block 196 len 12288 data: 6 8 9 17/30 Flush 18/30 Write block 431 len 12288 data: 6 6 4 19/30 Read block 264 len 4096 20/30 Write block 43 len 20480 data: 5 6 5 5 6 21/30 Flush 22/30 Write block 309 len 16384 data: 5 7 6 5 23/30 Write block 158 len 24576 data: 8 10 8 8 7 6 24/30 Read block 111 len 12288 25/30 Write block 506 len 24576 data: 6 4 4 4 4 4 26/30 Read block 251 len 12288 27/30 Read block 177 len 20480 28/30 Read block 23 len 32768 29/30 Write block 450 len 16384 data: 9 7 5 6 30/30 Write block 316 len 32768 data: 6 6 7 10 9 9 10 10 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:30:58.886362436Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:30:58.886522076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1432,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 121 121 121 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 70 74 74 119 127 127 F F F <--- 3 060-079 72 74 74 121 125 125 F F F <--- 4 080-099 71 71 71 120 120 120 F F F 5 100-119 73 73 73 122 122 122 F F F 6 120-139 73 73 73 122 122 122 F F F 7 140-159 69 69 69 116 116 116 F T T <--- 8 160-179 69 69 69 116 116 116 F T T <--- 9 180-199 73 74 74 122 126 126 F F F <--- 10 200-219 64 74 74 104 123 123 F F F <--- 11 220-239 67 67 67 111 111 111 F F F 12 240-259 70 70 70 119 119 119 F F F 13 260-279 73 74 74 122 124 124 F F F <--- 14 280-299 72 72 72 121 121 121 F F F 15 300-319 71 71 71 120 120 120 F T T <--- 16 320-339 71 71 71 120 120 120 F T T <--- 17 340-359 73 73 73 122 122 122 F F F 18 360-379 73 73 73 122 122 122 F F F 19 380-399 71 74 74 120 126 126 F F F <--- 20 400-419 73 73 73 122 122 122 F F F 21 420-439 73 74 74 122 127 127 F F F <--- 22 440-459 70 70 70 118 118 118 F T T <--- 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 73 73 73 122 122 122 F T T <--- 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 73 73 73 122 122 122 F F F Max gen: 74, Max flush: 127 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 121 121 121 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 74 74 74 127 127 127 F F F 3 060-079 74 74 74 125 125 125 F F F 4 080-099 71 71 71 120 120 120 F F F 5 100-119 73 73 73 122 122 122 F F F 6 120-139 73 73 73 122 122 122 F F F 7 140-159 75 75 75 128 128 128 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 74 74 74 126 126 126 F F F 10 200-219 74 74 74 123 123 123 F F F 11 220-239 67 67 67 111 111 111 F F F 12 240-259 70 70 70 119 119 119 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 73 73 73 122 122 122 F F F 18 360-379 73 73 73 122 122 122 F F F 19 380-399 74 74 74 126 126 126 F F F 20 400-419 73 73 73 122 122 122 F F F 21 420-439 74 74 74 127 127 127 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 66 66 66 108 108 108 F F F 29 580-599 73 73 73 122 122 122 F F F Max gen: 75, Max flush: 128 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.146803816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155131378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155413469Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155441008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155449657Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155461546Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Crucible 1c5c32f3-cb73-466c-ad54-66c2c582506f has session id: 915ebc55-76e8-4567-8c1e-b6cf95e442ff","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155836292Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Upstairs opts: Upstairs UUID: 1c5c32f3-cb73-466c-ad54-66c2c582506f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155906947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Crucible stats registered with UUID: 1c5c32f3-cb73-466c-ad54-66c2c582506f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.155920616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:03.155929925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156076006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156105214Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156116163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156286332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156303851Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"1c5c32f3-cb73-466c-ad54-66c2c582506f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.15632206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156331169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"0","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156340928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"1","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156350598Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"2","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156360117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156370146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156380146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:53983)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156428433Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:51204)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156450041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:54680)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.156538305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157118767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"2","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157144355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"0","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157194512Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"1","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157444526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157454305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[0]R flush_numbers[0..12]: [121, 102, 127, 125, 120, 122, 122, 128, 128, 126, 123, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157590046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[0]R generation[0..12]: [72, 63, 74, 74, 71, 73, 73, 75, 75, 74, 74, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157601215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157612275Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[1]R flush_numbers[0..12]: [121, 102, 127, 125, 120, 122, 122, 128, 128, 126, 123, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157623144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[1]R generation[0..12]: [72, 63, 74, 74, 71, 73, 73, 75, 75, 74, 74, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157633523Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157644423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[2]R flush_numbers[0..12]: [121, 102, 127, 125, 120, 122, 122, 128, 128, 126, 123, 111]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157655122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[2]R generation[0..12]: [72, 63, 74, 74, 71, 73, 73, 75, 75, 74, 74, 67]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157667491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.15767866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.15768835Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157698119Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157707588Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157717038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"":"downstairs","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157726837Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157735567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"0","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157745146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"1","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157754855Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"client":"2","":"downstairs","session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157764925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157773934Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"1c5c32f3-cb73-466c-ad54-66c2c582506f is now active with session: 915ebc55-76e8-4567-8c1e-b6cf95e442ff","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157784203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157793083Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"session_id":"915ebc55-76e8-4567-8c1e-b6cf95e442ff"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157802302Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Activated sub_volume 1c5c32f3-cb73-466c-ad54-66c2c582506f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157810332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.157820121Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.158086484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"task":"crutest"} 01/30 Write block 413 len 40960 data: 7 6 7 5 4 5 5 5 7 9 02/30 Flush 03/30 Write block 154 len 24576 data: 9 6 8 9 9 11 04/30 Write block 84 len 32768 data: 4 3 3 3 3 5 6 6 05/30 Read block 152 len 32768 06/30 Write block 112 len 36864 data: 6 5 4 4 6 7 8 10 10 07/30 Read block 191 len 16384 08/30 Write block 570 len 8192 data: 7 7 09/30 Write block 66 len 8192 data: 4 5 10/30 Flush 11/30 Flush 12/30 Flush 13/30 Write block 427 len 36864 data: 7 5 5 6 7 7 5 6 7 14/30 Write block 144 len 12288 data: 7 7 8 15/30 Write block 182 len 36864 data: 9 7 8 8 7 8 8 7 7 16/30 Read block 523 len 24576 17/30 Write block 503 len 12288 data: 5 5 6 18/30 Read block 297 len 12288 19/30 Read block 483 len 12288 20/30 Read block 233 len 40960 21/30 Read block 490 len 40960 22/30 Write block 157 len 40960 data: 10 10 12 9 9 8 7 5 5 6 23/30 Read block 352 len 24576 24/30 Read block 151 len 12288 25/30 Read block 115 len 40960 26/30 Read block 172 len 40960 27/30 Read block 18 len 40960 28/30 Read block 1 len 36864 29/30 Write block 215 len 28672 data: 14 12 9 8 5 5 4 30/30 Read block 80 len 24576 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 28 0 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1001)) Some(JobId(1011)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:03.167579609Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:03.167750478Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1440,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 121 121 121 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 74 74 74 127 127 127 F F F 3 060-079 76 74 76 130 125 130 F F F <--- 4 080-099 76 71 76 130 120 130 F F F <--- 5 100-119 76 73 76 130 122 130 F F F <--- 6 120-139 76 73 76 130 122 130 F F F <--- 7 140-159 76 75 76 130 128 130 T F T <--- 8 160-179 75 75 75 128 128 128 T F T <--- 9 180-199 74 74 74 126 126 126 T F T <--- 10 200-219 74 74 74 123 123 123 T F T <--- 11 220-239 67 67 67 111 111 111 T F T <--- 12 240-259 70 70 70 119 119 119 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 73 73 73 122 122 122 F F F 18 360-379 73 73 73 122 122 122 F F F 19 380-399 74 74 74 126 126 126 F F F 20 400-419 76 76 76 129 129 129 F F F 21 420-439 76 76 76 129 129 129 T F T <--- 22 440-459 75 75 75 128 128 128 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 75 75 75 128 128 128 T F T <--- 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 76 66 76 130 108 130 F F F <--- 29 580-599 73 73 73 122 122 122 F F F Max gen: 76, Max flush: 130 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 121 121 121 F F F 1 020-039 63 63 63 102 102 102 F F F 2 040-059 74 74 74 127 127 127 F F F 3 060-079 76 76 76 130 130 130 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 76 76 76 130 130 130 F F F 6 120-139 76 76 76 130 130 130 F F F 7 140-159 77 77 77 131 131 131 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 77 77 77 131 131 131 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 77 77 77 131 131 131 F F F 12 240-259 70 70 70 119 119 119 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 73 73 73 122 122 122 F F F 18 360-379 73 73 73 122 122 122 F F F 19 380-399 74 74 74 126 126 126 F F F 20 400-419 76 76 76 129 129 129 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 77 77 77 131 131 131 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 76 76 76 130 130 130 F F F 29 580-599 73 73 73 122 122 122 F F F Max gen: 77, Max flush: 131 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.420049452Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428250202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428530924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428555072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428563852Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428575571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Crucible bce7e336-4213-4352-993b-13bd36c2218e has session id: a8e4e5a0-2eef-4161-aa57-f9bceda750d5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.42888771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Upstairs opts: Upstairs UUID: bce7e336-4213-4352-993b-13bd36c2218e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428928858Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Crucible stats registered with UUID: bce7e336-4213-4352-993b-13bd36c2218e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.428939417Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:07.428948616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429125325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429153703Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429164692Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429269415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429286314Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"bce7e336-4213-4352-993b-13bd36c2218e active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.42935463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429364269Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"0","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429374158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"1","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429383938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"2","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429393677Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429403716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429432515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39516)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429448423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34384)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429519499Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39175)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.429580785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430148327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"2","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430224332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"1","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430274539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"0","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430470956Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430516533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[0]R flush_numbers[0..12]: [121, 102, 127, 130, 130, 130, 130, 131, 131, 131, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.43056464Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[0]R generation[0..12]: [72, 63, 74, 76, 76, 76, 76, 77, 77, 77, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430576579Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430587819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[1]R flush_numbers[0..12]: [121, 102, 127, 130, 130, 130, 130, 131, 131, 131, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430598598Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[1]R generation[0..12]: [72, 63, 74, 76, 76, 76, 76, 77, 77, 77, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430609047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430620186Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[2]R flush_numbers[0..12]: [121, 102, 127, 130, 130, 130, 130, 131, 131, 131, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430630816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[2]R generation[0..12]: [72, 63, 74, 76, 76, 76, 76, 77, 77, 77, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430641325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430652384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430661794Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430671473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430680892Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430690312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"":"downstairs","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430702581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.43071149Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"0","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.43072123Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"1","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430730849Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"client":"2","":"downstairs","session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430741028Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430750318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"bce7e336-4213-4352-993b-13bd36c2218e is now active with session: a8e4e5a0-2eef-4161-aa57-f9bceda750d5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430760167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-09-18T03:31:07.430769067Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"session_id":"a8e4e5a0-2eef-4161-aa57-f9bceda750d5"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430783546Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Activated sub_volume bce7e336-4213-4352-993b-13bd36c2218e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430791565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.430799775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.43102935Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"task":"crutest"} 01/30 Write block 197 len 8192 data: 9 10 02/30 Read block 214 len 4096 03/30 Write block 515 len 36864 data: 8 8 7 7 5 5 4 3 3 04/30 Read block 136 len 32768 05/30 Read block 518 len 16384 06/30 Write block 61 len 20480 data: 5 5 8 6 6 07/30 Read block 84 len 24576 08/30 Read block 427 len 32768 09/30 Write block 504 len 4096 data: 6 10/30 Read block 525 len 8192 11/30 Flush 12/30 Write block 14 len 40960 data: 6 5 4 5 5 6 7 8 7 7 13/30 Write block 348 len 32768 data: 7 7 7 7 7 6 6 5 14/30 Write block 16 len 16384 data: 5 6 6 7 15/30 Read block 104 len 40960 16/30 Read block 454 len 8192 17/30 Read block 243 len 32768 18/30 Read block 361 len 8192 19/30 Write block 250 len 20480 data: 8 7 6 6 5 20/30 Write block 389 len 32768 data: 7 9 10 10 10 9 8 6 21/30 Flush 22/30 Flush 23/30 Write block 249 len 24576 data: 7 9 8 7 7 6 24/30 Read block 140 len 20480 25/30 Read block 167 len 20480 26/30 Write block 61 len 32768 data: 6 6 9 7 7 5 6 6 27/30 Write block 582 len 4096 data: 4 28/30 Write block 342 len 36864 data: 9 8 8 8 9 9 8 8 8 29/30 Read block 260 len 24576 30/30 Write block 134 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 29 4 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:07.440037407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:07.440263262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1448,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 72 78 133 121 133 F F F <--- 1 020-039 78 63 78 133 102 133 F F F <--- 2 040-059 74 74 74 127 127 127 F F F 3 060-079 78 76 78 132 130 132 T F T <--- 4 080-099 76 76 76 130 130 130 F F F 5 100-119 76 76 76 130 130 130 F F F 6 120-139 76 76 76 130 130 130 T F T <--- 7 140-159 77 77 77 131 131 131 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 78 77 78 132 131 132 F T F <--- 10 200-219 77 77 77 131 131 131 F F F 11 220-239 77 77 77 131 131 131 F F F 12 240-259 78 70 78 133 119 133 T F T <--- 13 260-279 74 74 74 124 124 124 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 78 73 78 133 122 133 T F T <--- 18 360-379 73 73 73 122 122 122 F F F 19 380-399 78 74 78 133 126 133 F F F <--- 20 400-419 76 76 76 129 129 129 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 78 77 78 132 131 132 F T F <--- 26 520-539 78 71 78 132 120 132 F T F <--- 27 540-559 69 69 69 116 116 116 F F F 28 560-579 76 76 76 130 130 130 F F F 29 580-599 73 73 73 122 122 122 T F T <--- Max gen: 78, Max flush: 133 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 133 133 133 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 74 74 74 127 127 127 F F F 3 060-079 79 79 79 134 134 134 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 76 76 76 130 130 130 F F F 6 120-139 79 79 79 134 134 134 F F F 7 140-159 77 77 77 131 131 131 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 78 78 78 132 132 132 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 77 77 77 131 131 131 F F F 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 79 79 79 134 134 134 F F F 18 360-379 73 73 73 122 122 122 F F F 19 380-399 78 78 78 133 133 133 F F F 20 400-419 76 76 76 129 129 129 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 71 71 71 120 120 120 F F F 24 480-499 63 63 63 102 102 102 F F F 25 500-519 78 78 78 132 132 132 F F F 26 520-539 78 78 78 132 132 132 F F F 27 540-559 69 69 69 116 116 116 F F F 28 560-579 76 76 76 130 130 130 F F F 29 580-599 79 79 79 134 134 134 F F F Max gen: 79, Max flush: 134 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.692580006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:11.69981732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.699865987Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.70695817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707208824Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707234832Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707243701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707255581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Crucible c51cc699-dc05-4e8a-80ad-cdc0edd94b37 has session id: c66d6f23-94ab-4e27-a8b3-e4a511d0ba12","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707613717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Upstairs opts: Upstairs UUID: c51cc699-dc05-4e8a-80ad-cdc0edd94b37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707642195Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Crucible stats registered with UUID: c51cc699-dc05-4e8a-80ad-cdc0edd94b37","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707653384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:11.707662814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707801675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707834012Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707845172Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707981963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.707997732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"c51cc699-dc05-4e8a-80ad-cdc0edd94b37 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708031899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708041389Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"0","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708057668Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"1","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708067547Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"2","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708077286Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708087316Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708097515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"ds_connection connected from Ok(127.0.0.1:34330)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.70817563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"ds_connection connected from Ok(127.0.0.1:33157)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708216477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"ds_connection connected from Ok(127.0.0.1:61277)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708235516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708818078Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"1","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708843286Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"2","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.708857835Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"0","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709126447Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709142516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[0]R flush_numbers[0..12]: [133, 133, 127, 134, 130, 130, 134, 131, 131, 132, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709193363Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[0]R generation[0..12]: [78, 78, 74, 79, 76, 76, 79, 77, 77, 78, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709204902Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709218261Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[1]R flush_numbers[0..12]: [133, 133, 127, 134, 130, 130, 134, 131, 131, 132, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709229061Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[1]R generation[0..12]: [78, 78, 74, 79, 76, 76, 79, 77, 77, 78, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.70923966Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709250769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[2]R flush_numbers[0..12]: [133, 133, 127, 134, 130, 130, 134, 131, 131, 132, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709261439Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[2]R generation[0..12]: [78, 78, 74, 79, 76, 76, 79, 77, 77, 78, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709271898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709283007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709292517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709302226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709311735Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709321375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"":"downstairs","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709331084Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709339933Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"0","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709349873Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"1","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709359652Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"client":"2","":"downstairs","session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709371431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709380541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"c51cc699-dc05-4e8a-80ad-cdc0edd94b37 is now active with session: c66d6f23-94ab-4e27-a8b3-e4a511d0ba12","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.70939069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.70939961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"session_id":"c66d6f23-94ab-4e27-a8b3-e4a511d0ba12"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709408729Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Activated sub_volume c51cc699-dc05-4e8a-80ad-cdc0edd94b37","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709416818Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709424928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.709743797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} 01/30 Write block 379 len 40960 data: 2 2 2 3 3 3 4 5 6 7 02/30 Read block 516 len 20480 03/30 Read block 290 len 32768 04/30 Write block 41 len 8192 data: 3 5 05/30 Write block 59 len 12288 data: 5 5 7 06/30 Write block 479 len 32768 data: 8 7 8 8 7 7 8 6 07/30 Read block 577 len 28672 08/30 Read block 365 len 24576 09/30 Read block 129 len 28672 10/30 Read block 370 len 16384 11/30 Write block 139 len 8192 data: 6 5 12/30 Read block 398 len 12288 13/30 Read block 245 len 12288 14/30 Write block 451 len 40960 data: 8 6 7 6 5 6 5 5 7 7 15/30 Write block 176 len 28672 data: 6 8 7 8 8 10 10 16/30 Write block 6 len 28672 data: 10 10 11 7 8 7 7 17/30 Write block 505 len 28672 data: 7 7 5 5 5 5 5 18/30 Read block 35 len 12288 19/30 Read block 199 len 28672 20/30 Write block 450 len 28672 data: 10 9 7 8 7 6 7 21/30 Read block 159 len 24576 22/30 Read block 381 len 12288 23/30 Read block 491 len 32768 24/30 Read block 221 len 4096 25/30 Read block 336 len 16384 26/30 Write block 288 len 20480 data: 8 7 8 8 5 27/30 Write block 197 len 8192 data: 10 11 28/30 Write block 553 len 40960 data: 7 7 9 10 9 10 10 11 10 9 29/30 Write block 101 len 12288 data: 11 11 11 30/30 Read block 429 len 32768 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 15 0 0 15 Done 15 30 30 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:11.7185175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:11.718680649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1456,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 133 133 133 F T T <--- 1 020-039 78 78 78 133 133 133 F F F 2 040-059 74 74 74 127 127 127 T T T 3 060-079 79 79 79 134 134 134 T T T 4 080-099 76 76 76 130 130 130 F F F 5 100-119 76 76 76 130 130 130 F T T <--- 6 120-139 79 79 79 134 134 134 T T T 7 140-159 77 77 77 131 131 131 T T T 8 160-179 77 77 77 131 131 131 T T T 9 180-199 78 78 78 132 132 132 T T T 10 200-219 77 77 77 131 131 131 F F F 11 220-239 77 77 77 131 131 131 F F F 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 72 72 72 121 121 121 F T T <--- 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 79 79 79 134 134 134 F F F 18 360-379 73 73 73 122 122 122 T T T 19 380-399 78 78 78 133 133 133 T T T 20 400-419 76 76 76 129 129 129 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 75 75 75 128 128 128 T T T 23 460-479 71 71 71 120 120 120 T T T 24 480-499 63 63 63 102 102 102 T T T 25 500-519 78 78 78 132 132 132 F T T <--- 26 520-539 78 78 78 132 132 132 F F F 27 540-559 69 69 69 116 116 116 F T T <--- 28 560-579 76 76 76 130 130 130 F T T <--- 29 580-599 79 79 79 134 134 134 F F F Max gen: 79, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 200d199 < 9, 201a201 > 11, 452a453 > 10, 453a455 > 7, 455d456 < 6, 457a459 > 7, 459d460 < 6, 461d461 < 5, diff found after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 80 80 80 135 135 135 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 80 80 80 135 135 135 F F F 6 120-139 80 80 80 135 135 135 F F F 7 140-159 80 80 80 135 135 135 F F F 8 160-179 80 80 80 135 135 135 F F F 9 180-199 80 80 80 135 135 135 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 77 77 77 131 131 131 F F F 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 F F F 17 340-359 79 79 79 134 134 134 F F F 18 360-379 80 80 80 135 135 135 F F F 19 380-399 80 80 80 135 135 135 F F F 20 400-419 76 76 76 129 129 129 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 80 80 80 135 135 135 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 80 80 80 135 135 135 F F F 25 500-519 80 80 80 135 135 135 F F F 26 520-539 78 78 78 132 132 132 F F F 27 540-559 80 80 80 135 135 135 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 79 79 79 134 134 134 F F F Max gen: 80, Max flush: 135 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.004248335Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.012451115Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.012726607Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.012760885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.012770324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.012782423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Crucible 89da0bb4-8f14-4035-8f79-090ff2d73b69 has session id: 2b9e611e-8ce5-46d2-b573-ea75033068fb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013095423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Upstairs opts: Upstairs UUID: 89da0bb4-8f14-4035-8f79-090ff2d73b69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.01313476Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Crucible stats registered with UUID: 89da0bb4-8f14-4035-8f79-090ff2d73b69","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.01314621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:16.013155489Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013307299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013335807Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013346716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013480638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013496227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"89da0bb4-8f14-4035-8f79-090ff2d73b69 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013531984Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013541634Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"0","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013551663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"1","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013561492Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"2","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013571022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013580881Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.01359355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:54396)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013627808Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55144)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013638747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:44234)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.013649226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014335231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"2","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.01435857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"0","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014419626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"1","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014646071Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.01465576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[0]R flush_numbers[0..12]: [135, 133, 135, 135, 130, 135, 135, 135, 135, 135, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014712027Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[0]R generation[0..12]: [80, 78, 80, 80, 76, 80, 80, 80, 80, 80, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014723116Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014734365Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[1]R flush_numbers[0..12]: [135, 133, 135, 135, 130, 135, 135, 135, 135, 135, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014745184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[1]R generation[0..12]: [80, 78, 80, 80, 76, 80, 80, 80, 80, 80, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014755644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014766613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[2]R flush_numbers[0..12]: [135, 133, 135, 135, 130, 135, 135, 135, 135, 135, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014779982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[2]R generation[0..12]: [80, 78, 80, 80, 76, 80, 80, 80, 80, 80, 77, 77]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014790431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014801421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Max found gen is 81","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.01481103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Generation requested: 82 >= found:81","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014820839Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014830159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014839578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"":"downstairs","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014849148Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014857857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"0","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014867586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"1","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014877256Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"client":"2","":"downstairs","session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014887185Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014896444Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"89da0bb4-8f14-4035-8f79-090ff2d73b69 is now active with session: 2b9e611e-8ce5-46d2-b573-ea75033068fb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014906534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014915283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"session_id":"2b9e611e-8ce5-46d2-b573-ea75033068fb"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014925972Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Activated sub_volume 89da0bb4-8f14-4035-8f79-090ff2d73b69","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014934412Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.014942661Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.015237732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"task":"crutest"} 01/30 Write block 226 len 16384 data: 4 6 6 6 02/30 Write block 539 len 40960 data: 5 6 6 6 7 8 9 7 10 9 03/30 Flush 04/30 Write block 503 len 8192 data: 6 7 05/30 Write block 535 len 16384 data: 4 5 4 5 06/30 Read block 428 len 12288 07/30 Write block 229 len 20480 data: 7 5 4 4 4 08/30 Read block 104 len 24576 09/30 Read block 296 len 32768 10/30 Read block 258 len 8192 11/30 Write block 490 len 40960 data: 8 5 7 8 9 8 8 9 9 10 12/30 Write block 400 len 24576 data: 5 5 7 8 7 7 13/30 Read block 469 len 40960 14/30 Read block 300 len 16384 15/30 Read block 214 len 20480 16/30 Write block 359 len 40960 data: 8 7 7 9 10 11 9 11 10 10 17/30 Flush 18/30 Write block 354 len 32768 data: 7 6 6 8 6 9 8 8 19/30 Write block 329 len 32768 data: 9 8 7 8 8 5 6 8 20/30 Read block 292 len 28672 21/30 Write block 131 len 40960 data: 8 8 8 11 9 7 8 7 7 6 22/30 Read block 35 len 28672 23/30 Read block 561 len 28672 24/30 Read block 197 len 40960 25/30 Read block 522 len 20480 26/30 Read block 197 len 8192 27/30 Write block 122 len 20480 data: 9 10 10 9 8 28/30 Write block 347 len 8192 data: 10 9 29/30 Write block 538 len 32768 data: 6 6 7 7 7 8 9 10 30/30 Write block 417 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 5 Done Sent Sent false 1027 Acked Write 2 Done Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 4 36 Done 28 0 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:16.023982927Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:16.024154056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1467,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 80 80 80 135 135 135 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 80 80 80 135 135 135 F F F 6 120-139 80 80 80 135 135 135 T F T <--- 7 140-159 80 80 80 135 135 135 T F T <--- 8 160-179 80 80 80 135 135 135 F F F 9 180-199 80 80 80 135 135 135 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 82 77 82 137 131 137 F F F <--- 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 75 75 75 128 128 128 T F T <--- 17 340-359 82 79 82 137 134 137 T F T <--- 18 360-379 82 80 82 137 135 137 T F T <--- 19 380-399 80 80 80 135 135 135 F F F 20 400-419 82 76 82 137 129 137 T F T <--- 21 420-439 77 77 77 131 131 131 F F F 22 440-459 80 80 80 135 135 135 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 82 80 82 137 135 137 F F F <--- 25 500-519 82 80 82 137 135 137 F F F <--- 26 520-539 82 78 82 137 132 137 T F T <--- 27 540-559 82 80 82 136 135 136 T F T <--- 28 560-579 80 80 80 135 135 135 F F F 29 580-599 79 79 79 134 134 134 F F F Max gen: 82, Max flush: 137 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 80 80 80 135 135 135 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 80 80 80 135 135 135 F F F 6 120-139 83 83 83 138 138 138 F F F 7 140-159 83 83 83 138 138 138 F F F 8 160-179 80 80 80 135 135 135 F F F 9 180-199 80 80 80 135 135 135 F F F 10 200-219 77 77 77 131 131 131 F F F 11 220-239 82 82 82 137 137 137 F F F 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 83 83 83 138 138 138 F F F 18 360-379 83 83 83 138 138 138 F F F 19 380-399 80 80 80 135 135 135 F F F 20 400-419 83 83 83 138 138 138 F F F 21 420-439 77 77 77 131 131 131 F F F 22 440-459 80 80 80 135 135 135 F F F 23 460-479 80 80 80 135 135 135 F F F 24 480-499 82 82 82 137 137 137 F F F 25 500-519 82 82 82 137 137 137 F F F 26 520-539 83 83 83 138 138 138 F F F 27 540-559 83 83 83 138 138 138 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 79 79 79 134 134 134 F F F Max gen: 83, Max flush: 138 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.276383114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.284620032Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.284904633Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.284931902Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.284940581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.28495222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Crucible 0f3a8389-5819-4701-8629-6f4d4d37b8e6 has session id: c5dd799b-72ab-48a0-b8b3-b50527ae4ce9","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285273239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Upstairs opts: Upstairs UUID: 0f3a8389-5819-4701-8629-6f4d4d37b8e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285306867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Crucible stats registered with UUID: 0f3a8389-5819-4701-8629-6f4d4d37b8e6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285318166Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:20.285327796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285476316Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285491485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285501454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285622486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285634595Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"0f3a8389-5819-4701-8629-6f4d4d37b8e6 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285665763Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285675293Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"0","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285684942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"1","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285694521Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"2","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285704101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.28571421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285724509Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44167)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285799505Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52755)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285929486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42908)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.285954884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286425503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"2","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286453152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"0","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.28647663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"1","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286760331Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286802159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[0]R flush_numbers[0..12]: [135, 133, 135, 135, 130, 135, 138, 138, 135, 135, 131, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286851085Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[0]R generation[0..12]: [80, 78, 80, 80, 76, 80, 83, 83, 80, 80, 77, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286862255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286873494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[1]R flush_numbers[0..12]: [135, 133, 135, 135, 130, 135, 138, 138, 135, 135, 131, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286884203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[1]R generation[0..12]: [80, 78, 80, 80, 76, 80, 83, 83, 80, 80, 77, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286894623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286905562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[2]R flush_numbers[0..12]: [135, 133, 135, 135, 130, 135, 138, 138, 135, 135, 131, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286916171Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[2]R generation[0..12]: [80, 78, 80, 80, 76, 80, 83, 83, 80, 80, 77, 82]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.28692653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.28693742Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286946779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286958448Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286967878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286977277Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"":"downstairs","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286987016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.286995806Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"0","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287005275Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"1","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287014965Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"client":"2","":"downstairs","session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287024774Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287033903Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"0f3a8389-5819-4701-8629-6f4d4d37b8e6 is now active with session: c5dd799b-72ab-48a0-b8b3-b50527ae4ce9","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287043923Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287052682Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"session_id":"c5dd799b-72ab-48a0-b8b3-b50527ae4ce9"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287061792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Activated sub_volume 0f3a8389-5819-4701-8629-6f4d4d37b8e6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287069791Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.28707786Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.287345203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"task":"crutest"} 01/30 Write block 477 len 40960 data: 8 8 9 8 9 9 8 8 9 7 02/30 Write block 400 len 24576 data: 6 6 8 9 8 8 03/30 Read block 194 len 40960 04/30 Read block 292 len 16384 05/30 Read block 120 len 24576 06/30 Read block 491 len 8192 07/30 Read block 356 len 32768 08/30 Write block 482 len 4096 data: 10 09/30 Write block 514 len 28672 data: 6 9 9 8 8 6 6 10/30 Read block 451 len 24576 11/30 Read block 534 len 40960 12/30 Write block 533 len 20480 data: 4 3 5 6 5 13/30 Write block 108 len 20480 data: 7 6 6 6 7 14/30 Write block 518 len 12288 data: 9 7 7 15/30 Read block 434 len 8192 16/30 Flush 17/30 Write block 556 len 16384 data: 11 10 11 11 18/30 Read block 325 len 24576 19/30 Read block 401 len 24576 20/30 Flush 21/30 Write block 69 len 16384 data: 11 11 10 11 22/30 Write block 447 len 12288 data: 9 10 10 23/30 Flush 24/30 Write block 128 len 16384 data: 8 7 6 9 25/30 Write block 231 len 32768 data: 5 5 5 5 6 6 7 7 26/30 Read block 158 len 36864 27/30 Write block 411 len 40960 data: 8 7 8 7 8 6 6 6 6 6 28/30 Write block 598 len 8192 data: 7 5 29/30 Write block 211 len 36864 data: 15 15 14 14 15 13 10 9 6 30/30 Write block 381 len 32768 data: 3 4 4 4 5 6 7 8 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 10 Done Sent Sent false 1027 Acked Write 2 Done Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 4 32 Done 28 4 26 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:20.295372255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:20.295558143Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1475,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 84 80 84 141 135 141 F F F <--- 4 080-099 76 76 76 130 130 130 F F F 5 100-119 84 80 84 139 135 139 F F F <--- 6 120-139 83 83 83 138 138 138 T F T <--- 7 140-159 83 83 83 138 138 138 F F F 8 160-179 80 80 80 135 135 135 F F F 9 180-199 80 80 80 135 135 135 F F F 10 200-219 77 77 77 131 131 131 T F T <--- 11 220-239 82 82 82 137 137 137 T F T <--- 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 83 83 83 138 138 138 F F F 18 360-379 83 83 83 138 138 138 F F F 19 380-399 80 80 80 135 135 135 T F T <--- 20 400-419 84 83 84 139 138 139 T T T <--- 21 420-439 77 77 77 131 131 131 T F T <--- 22 440-459 84 80 84 141 135 141 F F F <--- 23 460-479 84 80 84 139 135 139 F T F <--- 24 480-499 84 82 84 139 137 139 F T F <--- 25 500-519 84 82 84 139 137 139 F F F <--- 26 520-539 84 83 84 139 138 139 F F F <--- 27 540-559 84 83 84 140 138 140 F F F <--- 28 560-579 80 80 80 135 135 135 F F F 29 580-599 79 79 79 134 134 134 T F T <--- Max gen: 84, Max flush: 141 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 84 84 84 141 141 141 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 84 84 84 139 139 139 F F F 6 120-139 85 85 85 142 142 142 F F F 7 140-159 83 83 83 138 138 138 F F F 8 160-179 80 80 80 135 135 135 F F F 9 180-199 80 80 80 135 135 135 F F F 10 200-219 85 85 85 142 142 142 F F F 11 220-239 85 85 85 142 142 142 F F F 12 240-259 79 79 79 134 134 134 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 75 75 75 128 128 128 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 83 83 83 138 138 138 F F F 18 360-379 83 83 83 138 138 138 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 85 85 85 142 142 142 F F F 21 420-439 85 85 85 142 142 142 F F F 22 440-459 84 84 84 141 141 141 F F F 23 460-479 84 84 84 139 139 139 F F F 24 480-499 84 84 84 139 139 139 F F F 25 500-519 84 84 84 139 139 139 F F F 26 520-539 84 84 84 139 139 139 F F F 27 540-559 84 84 84 140 140 140 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 85 85 85 142 142 142 F F F Max gen: 85, Max flush: 142 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.554270866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.562448928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.562753168Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.562781826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.562790145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.562801795Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Crucible 8742175e-41bb-42d2-b72d-4f58c98ca854 has session id: df64e519-7a2c-4979-b9fe-ccba3e553cd1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563118214Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Upstairs opts: Upstairs UUID: 8742175e-41bb-42d2-b72d-4f58c98ca854, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563139372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Crucible stats registered with UUID: 8742175e-41bb-42d2-b72d-4f58c98ca854","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563149272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:24.563158101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563296662Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563312021Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56332191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563464861Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56348063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"8742175e-41bb-42d2-b72d-4f58c98ca854 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563504168Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563515728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"0","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563525587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"1","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563535386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"2","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563544946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563555015Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563564864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37561)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.563608042Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62308)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56362416Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47947)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.5636345Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564197103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"2","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564274898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"0","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564320515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"1","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564477264Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564586047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[0]R flush_numbers[0..12]: [135, 133, 135, 141, 130, 139, 142, 138, 135, 135, 142, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564623395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[0]R generation[0..12]: [80, 78, 80, 84, 76, 84, 85, 83, 80, 80, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564637394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564648523Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[1]R flush_numbers[0..12]: [135, 133, 135, 141, 130, 139, 142, 138, 135, 135, 142, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564659232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[1]R generation[0..12]: [80, 78, 80, 84, 76, 84, 85, 83, 80, 80, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564669562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564680491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[2]R flush_numbers[0..12]: [135, 133, 135, 141, 130, 139, 142, 138, 135, 135, 142, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56469101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[2]R generation[0..12]: [80, 78, 80, 84, 76, 84, 85, 83, 80, 80, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56470167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564712659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564722128Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564731918Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564741237Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564750586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"":"downstairs","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564760486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564769305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"0","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564779095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"1","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564790294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"client":"2","":"downstairs","session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564800243Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564809473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"8742175e-41bb-42d2-b72d-4f58c98ca854 is now active with session: df64e519-7a2c-4979-b9fe-ccba3e553cd1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564819492Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564828101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"session_id":"df64e519-7a2c-4979-b9fe-ccba3e553cd1"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.564837171Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Activated sub_volume 8742175e-41bb-42d2-b72d-4f58c98ca854","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56484517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.56485327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.5651539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"task":"crutest"} 01/30 Write block 214 len 36864 data: 15 16 14 11 10 7 6 5 5 02/30 Read block 547 len 36864 03/30 Write block 523 len 4096 data: 4 04/30 Write block 475 len 32768 data: 8 8 9 9 10 9 10 11 05/30 Write block 149 len 36864 data: 12 10 11 11 9 10 7 9 11 06/30 Write block 77 len 32768 data: 6 4 4 4 4 3 3 5 07/30 Write block 484 len 20480 data: 9 10 8 7 9 08/30 Read block 340 len 8192 09/30 Write block 305 len 36864 data: 7 6 6 6 6 8 7 6 5 10/30 Write block 585 len 36864 data: 6 7 5 6 6 7 7 7 6 11/30 Write block 129 len 36864 data: 8 7 10 9 9 12 10 8 9 12/30 Read block 395 len 8192 13/30 Flush 14/30 Write block 234 len 28672 data: 6 7 7 8 8 8 8 15/30 Read block 234 len 8192 16/30 Read block 190 len 36864 17/30 Write block 529 len 12288 data: 3 5 5 18/30 Write block 400 len 4096 data: 7 19/30 Write block 28 len 20480 data: 4 4 4 4 4 20/30 Write block 134 len 16384 data: 13 11 9 10 21/30 Write block 21 len 32768 data: 9 8 8 6 5 5 4 5 22/30 Write block 29 len 28672 data: 5 5 5 5 4 3 2 23/30 Read block 586 len 40960 24/30 Write block 447 len 40960 data: 10 11 11 10 9 7 8 7 6 7 25/30 Write block 508 len 12288 data: 6 6 6 26/30 Read block 302 len 8192 27/30 Read block 573 len 16384 28/30 Write block 114 len 4096 data: 5 29/30 Write block 194 len 28672 data: 6 5 7 10 11 11 9 30/30 Write block 588 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) None Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:24.572794147Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:24.572946937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1483,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 78 78 78 133 133 133 T F T <--- 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 84 86 143 141 143 F F F <--- 4 080-099 86 76 86 143 130 143 F F F <--- 5 100-119 84 84 84 139 139 139 T F T <--- 6 120-139 86 85 86 143 142 143 T F T <--- 7 140-159 86 83 86 143 138 143 F F F <--- 8 160-179 80 80 80 135 135 135 F F F 9 180-199 80 80 80 135 135 135 T F T <--- 10 200-219 86 85 86 143 142 143 T T T <--- 11 220-239 86 85 86 143 142 143 T T T <--- 12 240-259 79 79 79 134 134 134 T F T <--- 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 86 75 86 143 128 143 F F F <--- 16 320-339 83 83 83 138 138 138 F F F 17 340-359 83 83 83 138 138 138 F F F 18 360-379 83 83 83 138 138 138 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 85 85 85 142 142 142 T F T <--- 21 420-439 85 85 85 142 142 142 F F F 22 440-459 84 84 84 141 141 141 T F T <--- 23 460-479 86 84 86 143 139 143 F F F <--- 24 480-499 86 84 86 143 139 143 F F F <--- 25 500-519 84 84 84 139 139 139 T F T <--- 26 520-539 86 84 86 143 139 143 T F T <--- 27 540-559 84 84 84 140 140 140 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 86 85 86 143 142 143 T F T <--- Max gen: 86, Max flush: 143 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 F F F 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 86 86 86 143 143 143 F F F 5 100-119 87 87 87 144 144 144 F F F 6 120-139 87 87 87 144 144 144 F F F 7 140-159 86 86 86 143 143 143 F F F 8 160-179 80 80 80 135 135 135 F F F 9 180-199 87 87 87 144 144 144 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 87 87 87 144 144 144 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 83 83 83 138 138 138 F F F 18 360-379 83 83 83 138 138 138 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 87 87 87 144 144 144 F F F 21 420-439 85 85 85 142 142 142 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 86 86 86 143 143 143 F F F 25 500-519 87 87 87 144 144 144 F F F 26 520-539 87 87 87 144 144 144 F F F 27 540-559 84 84 84 140 140 140 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 87 87 87 144 144 144 F F F Max gen: 87, Max flush: 144 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.839600848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:28.846849571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.846904858Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.853957574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854211187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854237795Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854249184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854261004Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Crucible 2cb021a6-c90a-4c91-b5b3-a4c24146fbb8 has session id: 665c6de9-b640-4919-90be-6a5f428eef89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854631999Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Upstairs opts: Upstairs UUID: 2cb021a6-c90a-4c91-b5b3-a4c24146fbb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854662327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Crucible stats registered with UUID: 2cb021a6-c90a-4c91-b5b3-a4c24146fbb8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854672637Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:28.854681776Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854835146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854849965Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.854859914Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855011134Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855027493Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"2cb021a6-c90a-4c91-b5b3-a4c24146fbb8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855044342Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855053192Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"0","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855062791Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"1","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.85507464Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"2","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.8550843Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855094559Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855104548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"ds_connection connected from Ok(127.0.0.1:62087)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855168764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"ds_connection connected from Ok(127.0.0.1:33321)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855284736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"ds_connection connected from Ok(127.0.0.1:64029)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855296266Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855926304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"0","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855938163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"2","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.855949183Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"1","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.85629779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856307409Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[0]R flush_numbers[0..12]: [135, 144, 135, 143, 143, 144, 144, 143, 135, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856322158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[0]R generation[0..12]: [80, 87, 80, 86, 86, 87, 87, 86, 80, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856332697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856343817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[1]R flush_numbers[0..12]: [135, 144, 135, 143, 143, 144, 144, 143, 135, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856356516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[1]R generation[0..12]: [80, 87, 80, 86, 86, 87, 87, 86, 80, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856366815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856377914Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[2]R flush_numbers[0..12]: [135, 144, 135, 143, 143, 144, 144, 143, 135, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856388604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[2]R generation[0..12]: [80, 87, 80, 86, 86, 87, 87, 86, 80, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856399063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856409992Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856419402Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856429241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.85643866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.8564481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"":"downstairs","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856457869Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856466599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"0","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856476678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"1","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856486677Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"client":"2","":"downstairs","session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856496807Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856507836Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"2cb021a6-c90a-4c91-b5b3-a4c24146fbb8 is now active with session: 665c6de9-b640-4919-90be-6a5f428eef89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856517925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856526705Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"session_id":"665c6de9-b640-4919-90be-6a5f428eef89"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856535884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Activated sub_volume 2cb021a6-c90a-4c91-b5b3-a4c24146fbb8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856543964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856552203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.856853793Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} 01/30 Write block 96 len 36864 data: 7 7 9 10 11 12 12 12 9 02/30 Flush 03/30 Write block 0 len 32768 data: 5 6 6 6 6 9 11 11 04/30 Read block 343 len 40960 05/30 Read block 331 len 36864 06/30 Write block 426 len 16384 data: 8 8 6 6 07/30 Flush 08/30 Write block 538 len 24576 data: 7 7 8 8 8 9 09/30 Read block 48 len 24576 10/30 Write block 168 len 16384 data: 5 6 6 7 11/30 Read block 88 len 28672 12/30 Read block 167 len 28672 13/30 Read block 378 len 16384 14/30 Read block 29 len 24576 15/30 Read block 208 len 16384 16/30 Read block 153 len 24576 17/30 Read block 4 len 40960 18/30 Read block 523 len 8192 19/30 Read block 215 len 4096 20/30 Write block 341 len 24576 data: 10 10 9 9 9 10 21/30 Read block 502 len 4096 22/30 Write block 427 len 4096 data: 9 23/30 Read block 415 len 36864 24/30 Write block 363 len 12288 data: 11 12 10 25/30 Flush 26/30 Write block 104 len 32768 data: 10 9 6 7 8 7 7 7 27/30 Read block 53 len 24576 28/30 Write block 489 len 36864 data: 8 9 6 8 9 10 9 9 10 29/30 Read block 328 len 28672 30/30 Read block 161 len 28672 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:28.86708548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:28.86724419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1491,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 88 88 135 146 146 F F F <--- 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 86 88 88 143 145 145 T F F <--- 5 100-119 87 88 88 144 145 145 T T T <--- 6 120-139 87 87 87 144 144 144 F F F 7 140-159 86 86 86 143 143 143 F F F 8 160-179 80 88 88 135 147 147 F F F <--- 9 180-199 87 87 87 144 144 144 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 87 87 87 144 144 144 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 83 88 88 138 147 147 F F F <--- 18 360-379 83 88 88 138 147 147 F F F <--- 19 380-399 85 85 85 142 142 142 F F F 20 400-419 87 87 87 144 144 144 F F F 21 420-439 85 88 88 142 147 147 F F F <--- 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 86 86 86 143 143 143 F T T <--- 25 500-519 87 87 87 144 144 144 F F F 26 520-539 87 88 88 144 147 147 F F F <--- 27 540-559 84 88 88 140 147 147 F F F <--- 28 560-579 80 80 80 135 135 135 F F F 29 580-599 87 87 87 144 144 144 F F F Max gen: 88, Max flush: 147 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 146 146 146 F F F 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 88 88 88 145 145 145 F F F 5 100-119 89 89 89 148 148 148 F F F 6 120-139 87 87 87 144 144 144 F F F 7 140-159 86 86 86 143 143 143 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 87 87 87 144 144 144 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 87 87 87 144 144 144 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 88 88 88 147 147 147 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 87 87 87 144 144 144 F F F 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 87 87 87 144 144 144 F F F 26 520-539 88 88 88 147 147 147 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 80 80 80 135 135 135 F F F 29 580-599 87 87 87 144 144 144 F F F Max gen: 89, Max flush: 148 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.120604325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:33.12782581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.127873847Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.134715447Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.13496436Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.134997558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135007427Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135019707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Crucible 63e402c7-44da-4045-9ca6-8a981c4b94d4 has session id: 3b522942-7b14-40f6-95db-85f402e96737","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135320787Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Upstairs opts: Upstairs UUID: 63e402c7-44da-4045-9ca6-8a981c4b94d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135357254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Crucible stats registered with UUID: 63e402c7-44da-4045-9ca6-8a981c4b94d4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135368464Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:33.135377883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135515464Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135544732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135556031Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135681893Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135696642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"63e402c7-44da-4045-9ca6-8a981c4b94d4 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.13572878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135738519Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"0","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135748369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"1","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135758198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"2","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135767867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135777737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135787736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"ds_connection connected from Ok(127.0.0.1:54057)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135929197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"ds_connection connected from Ok(127.0.0.1:48926)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135957525Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"ds_connection connected from Ok(127.0.0.1:51754)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.135969134Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136553776Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"1","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136581774Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"2","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.13664425Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"0","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136874195Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136914272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[0]R flush_numbers[0..12]: [146, 144, 135, 143, 145, 148, 144, 143, 147, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.1369428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[0]R generation[0..12]: [88, 87, 80, 86, 88, 89, 87, 86, 88, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136954669Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136966099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[1]R flush_numbers[0..12]: [146, 144, 135, 143, 145, 148, 144, 143, 147, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136977068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[1]R generation[0..12]: [88, 87, 80, 86, 88, 89, 87, 86, 88, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.136987737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137011826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[2]R flush_numbers[0..12]: [146, 144, 135, 143, 145, 148, 144, 143, 147, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137022765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[2]R generation[0..12]: [88, 87, 80, 86, 88, 89, 87, 86, 88, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137036164Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137047323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137056833Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137066792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137076321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137086041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"":"downstairs","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.13709583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.13710495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"0","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137114889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"1","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137125058Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"client":"2","":"downstairs","session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137135238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137144657Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"63e402c7-44da-4045-9ca6-8a981c4b94d4 is now active with session: 3b522942-7b14-40f6-95db-85f402e96737","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137154686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137163376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"session_id":"3b522942-7b14-40f6-95db-85f402e96737"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137172635Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Activated sub_volume 63e402c7-44da-4045-9ca6-8a981c4b94d4","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137180815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137191504Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.137486164Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} 01/30 Write block 166 len 8192 data: 7 6 02/30 Read block 21 len 24576 03/30 Read block 185 len 20480 04/30 Read block 148 len 40960 05/30 Flush 06/30 Write block 91 len 20480 data: 7 6 6 6 7 07/30 Read block 374 len 4096 08/30 Write block 524 len 24576 data: 3 4 5 5 3 4 09/30 Read block 352 len 16384 10/30 Read block 537 len 16384 11/30 Read block 519 len 40960 12/30 Flush 13/30 Write block 375 len 16384 data: 5 3 2 2 14/30 Read block 384 len 12288 15/30 Read block 547 len 28672 16/30 Write block 377 len 8192 data: 3 3 17/30 Read block 598 len 4096 18/30 Read block 115 len 16384 19/30 Write block 528 len 40960 data: 4 5 6 6 5 5 4 6 7 6 20/30 Write block 165 len 20480 data: 6 8 7 6 7 21/30 Read block 345 len 12288 22/30 Flush 23/30 Write block 408 len 4096 data: 6 24/30 Read block 352 len 40960 25/30 Write block 142 len 28672 data: 11 10 8 8 9 11 11 26/30 Read block 23 len 36864 27/30 Read block 88 len 36864 28/30 Write block 561 len 32768 data: 11 10 8 7 6 8 9 7 29/30 Read block 44 len 4096 30/30 Read block 581 len 28672 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 3 29 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:33.146557128Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:33.146729926Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1499,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 146 146 146 F F F 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 88 90 90 145 150 150 F F F <--- 5 100-119 89 89 89 148 148 148 F F F 6 120-139 87 87 87 144 144 144 F F F 7 140-159 86 86 86 143 143 143 F T T <--- 8 160-179 88 90 90 147 151 151 T F F <--- 9 180-199 87 87 87 144 144 144 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 87 87 87 144 144 144 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 88 90 90 147 151 151 F F F <--- 19 380-399 85 85 85 142 142 142 F F F 20 400-419 87 87 87 144 144 144 F T T <--- 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 87 87 87 144 144 144 F F F 26 520-539 88 90 90 147 151 151 F F F <--- 27 540-559 88 88 88 147 147 147 F F F 28 560-579 80 80 80 135 135 135 F T T <--- 29 580-599 87 87 87 144 144 144 F F F Max gen: 90, Max flush: 151 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 146 146 146 F F F 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 90 90 90 150 150 150 F F F 5 100-119 89 89 89 148 148 148 F F F 6 120-139 87 87 87 144 144 144 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 87 87 87 144 144 144 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 87 87 87 144 144 144 F F F 13 260-279 74 74 74 124 124 124 F F F 14 280-299 80 80 80 135 135 135 F F F 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 90 90 90 151 151 151 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 91 91 91 152 152 152 F F F 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 87 87 87 144 144 144 F F F 26 520-539 90 90 90 151 151 151 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 87 87 87 144 144 144 F F F Max gen: 91, Max flush: 152 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.394605231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403086503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403365085Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403390653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403399473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403411202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Crucible 29a544ab-9103-4b4e-a643-345b73bf66bd has session id: 165ff37f-6938-46bf-8b45-e130ff05e6a5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403711432Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Upstairs opts: Upstairs UUID: 29a544ab-9103-4b4e-a643-345b73bf66bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40374273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Crucible stats registered with UUID: 29a544ab-9103-4b4e-a643-345b73bf66bd","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403753799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:37.403763289Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40389223Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403923638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.403937507Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404064069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404077748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"29a544ab-9103-4b4e-a643-345b73bf66bd active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404101337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404110836Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"0","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404120575Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"1","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404130205Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"2","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404170782Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40419465Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40420561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53200)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404216139Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58265)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404226678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41620)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404237158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404864246Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"1","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404887575Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"2","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.404899074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"0","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405157057Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405187665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[0]R flush_numbers[0..12]: [146, 144, 135, 143, 150, 148, 144, 152, 151, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405241202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[0]R generation[0..12]: [88, 87, 80, 86, 90, 89, 87, 91, 90, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405252301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40526366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[1]R flush_numbers[0..12]: [146, 144, 135, 143, 150, 148, 144, 152, 151, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405274649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[1]R generation[0..12]: [88, 87, 80, 86, 90, 89, 87, 91, 90, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405285099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405295988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[2]R flush_numbers[0..12]: [146, 144, 135, 143, 150, 148, 144, 152, 151, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405306767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[2]R generation[0..12]: [88, 87, 80, 86, 90, 89, 87, 91, 90, 87, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405317257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405328246Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405337535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405347265Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405356584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405367733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"":"downstairs","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405377523Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405386392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"0","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405395951Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"1","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405405581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"client":"2","":"downstairs","session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40541553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.40542481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"29a544ab-9103-4b4e-a643-345b73bf66bd is now active with session: 165ff37f-6938-46bf-8b45-e130ff05e6a5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405434789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405443478Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"session_id":"165ff37f-6938-46bf-8b45-e130ff05e6a5"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405452668Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Activated sub_volume 29a544ab-9103-4b4e-a643-345b73bf66bd","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405460707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405468997Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.405788376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"task":"crutest"} 01/30 Write block 594 len 8192 data: 8 7 02/30 Read block 385 len 12288 03/30 Write block 257 len 40960 data: 4 4 3 4 6 4 3 4 4 5 04/30 Write block 285 len 20480 data: 7 7 6 9 8 05/30 Write block 126 len 16384 data: 9 9 9 9 06/30 Write block 192 len 32768 data: 5 6 7 6 8 11 12 12 07/30 Read block 127 len 16384 08/30 Flush 09/30 Write block 510 len 8192 data: 7 6 10/30 Read block 87 len 32768 11/30 Read block 44 len 20480 12/30 Read block 539 len 12288 13/30 Write block 246 len 20480 data: 7 8 8 8 10 14/30 Read block 569 len 40960 15/30 Read block 258 len 20480 16/30 Read block 33 len 28672 17/30 Read block 455 len 16384 18/30 Read block 400 len 12288 19/30 Read block 590 len 20480 20/30 Write block 264 len 36864 data: 5 5 6 5 7 6 5 5 6 21/30 Write block 241 len 28672 data: 6 7 6 6 7 8 9 22/30 Read block 72 len 40960 23/30 Read block 115 len 28672 24/30 Read block 236 len 28672 25/30 Read block 11 len 16384 26/30 Read block 82 len 32768 27/30 Write block 487 len 32768 data: 8 10 9 10 7 9 10 11 28/30 Read block 441 len 4096 29/30 Read block 505 len 20480 30/30 Write block 297 len 12288 data: 5 5 6 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Read 3 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:37.414448996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:37.414610235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1507,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 146 146 146 F F F 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 90 90 90 150 150 150 F F F 5 100-119 89 89 89 148 148 148 F F F 6 120-139 92 92 87 153 153 144 F F F <--- 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 87 153 153 144 F F F <--- 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 92 92 87 153 153 144 T T F <--- 13 260-279 92 92 74 153 153 124 T T F <--- 14 280-299 92 92 80 153 153 135 T T F <--- 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 90 90 90 151 151 151 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 91 91 91 152 152 152 F F F 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 89 89 89 148 148 148 T T F <--- 25 500-519 87 87 87 144 144 144 T T F <--- 26 520-539 90 90 90 151 151 151 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 92 92 87 153 153 144 F F F <--- Max gen: 92, Max flush: 153 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 146 146 146 F F F 1 020-039 87 87 87 144 144 144 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 90 90 90 150 150 150 F F F 5 100-119 89 89 89 148 148 148 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 93 93 93 154 154 154 F F F 13 260-279 93 93 93 154 154 154 F F F 14 280-299 93 93 93 154 154 154 F F F 15 300-319 86 86 86 143 143 143 F F F 16 320-339 83 83 83 138 138 138 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 90 90 90 151 151 151 F F F 19 380-399 85 85 85 142 142 142 F F F 20 400-419 91 91 91 152 152 152 F F F 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 86 86 86 143 143 143 F F F 24 480-499 93 93 93 154 154 154 F F F 25 500-519 93 93 93 154 154 154 F F F 26 520-539 90 90 90 151 151 151 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 92 92 92 153 153 153 F F F Max gen: 93, Max flush: 154 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.660113337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.668362175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.668643516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.668670044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.668684583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.668696333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Crucible 7e5a22fe-6186-4c09-8b1f-d0a1f88b4483 has session id: 2a7a2928-99a2-4006-b0f1-aaa15b0d0e82","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.6690344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Upstairs opts: Upstairs UUID: 7e5a22fe-6186-4c09-8b1f-d0a1f88b4483, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669056389Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Crucible stats registered with UUID: 7e5a22fe-6186-4c09-8b1f-d0a1f88b4483","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669066718Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:41.669075838Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669210479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669225518Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669235567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669402196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669417605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"7e5a22fe-6186-4c09-8b1f-d0a1f88b4483 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669453993Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669463462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"0","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669473262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"1","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669482751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"2","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.66949465Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669504879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669514819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"ds_connection connected from Ok(127.0.0.1:38462)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669558076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"ds_connection connected from Ok(127.0.0.1:62853)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669569145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"ds_connection connected from Ok(127.0.0.1:44065)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.669579545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670267479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"2","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670302727Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"0","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670313926Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"1","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670577749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670615556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[0]R flush_numbers[0..12]: [146, 144, 135, 143, 150, 148, 153, 152, 151, 153, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670661883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[0]R generation[0..12]: [88, 87, 80, 86, 90, 89, 92, 91, 90, 92, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670673003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670684072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[1]R flush_numbers[0..12]: [146, 144, 135, 143, 150, 148, 153, 152, 151, 153, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670697841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[1]R generation[0..12]: [88, 87, 80, 86, 90, 89, 92, 91, 90, 92, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.67070833Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.6707194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[2]R flush_numbers[0..12]: [146, 144, 135, 143, 150, 148, 153, 152, 151, 153, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670730159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[2]R generation[0..12]: [88, 87, 80, 86, 90, 89, 92, 91, 90, 92, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670740498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670751477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670760917Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670770666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670780056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670789395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"":"downstairs","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670799074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670807744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"0","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670817353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"1","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670827053Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"client":"2","":"downstairs","session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670836982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670847741Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"7e5a22fe-6186-4c09-8b1f-d0a1f88b4483 is now active with session: 2a7a2928-99a2-4006-b0f1-aaa15b0d0e82","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.67085773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.67086653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"session_id":"2a7a2928-99a2-4006-b0f1-aaa15b0d0e82"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670875689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Activated sub_volume 7e5a22fe-6186-4c09-8b1f-d0a1f88b4483","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670883769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.670891908Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.671196938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"task":"crutest"} 01/30 Write block 87 len 24576 data: 4 4 6 7 8 7 02/30 Read block 551 len 32768 03/30 Read block 325 len 4096 04/30 Write block 18 len 20480 data: 7 8 8 10 9 05/30 Write block 483 len 4096 data: 9 06/30 Write block 281 len 36864 data: 4 5 5 6 8 8 7 10 9 07/30 Write block 330 len 12288 data: 9 8 9 08/30 Write block 374 len 4096 data: 5 09/30 Read block 252 len 28672 10/30 Read block 526 len 32768 11/30 Read block 453 len 20480 12/30 Read block 556 len 32768 13/30 Read block 32 len 12288 14/30 Read block 573 len 8192 15/30 Read block 258 len 32768 16/30 Read block 515 len 16384 17/30 Write block 106 len 8192 data: 7 8 18/30 Flush 19/30 Write block 465 len 4096 data: 8 20/30 Write block 266 len 20480 data: 7 6 8 7 6 21/30 Flush 22/30 Write block 375 len 36864 data: 6 4 4 4 3 3 4 5 5 23/30 Write block 304 len 36864 data: 9 8 7 7 7 7 9 8 7 24/30 Write block 17 len 16384 data: 7 8 9 9 25/30 Write block 86 len 12288 data: 4 5 5 26/30 Read block 253 len 4096 27/30 Read block 95 len 16384 28/30 Write block 512 len 24576 data: 6 6 7 10 10 9 29/30 Read block 342 len 40960 30/30 Write block 307 len 32768 data: 8 8 8 10 9 8 6 6 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:41.679401979Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:41.679598336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1515,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 88 94 155 146 155 T F T <--- 1 020-039 94 87 94 155 144 155 T F T <--- 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 94 90 94 155 150 155 T F T <--- 5 100-119 94 89 94 155 148 155 F F F <--- 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 93 93 93 154 154 154 F F F 13 260-279 94 93 94 156 154 156 F F F <--- 14 280-299 94 93 94 155 154 155 F F F <--- 15 300-319 86 86 86 143 143 143 T F T <--- 16 320-339 94 83 94 155 138 155 F F F <--- 17 340-359 88 88 88 147 147 147 F F F 18 360-379 94 90 94 155 151 155 T F T <--- 19 380-399 85 85 85 142 142 142 T F T <--- 20 400-419 91 91 91 152 152 152 F F F 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 94 86 94 156 143 156 F F F <--- 24 480-499 94 93 94 155 154 155 F F F <--- 25 500-519 93 93 93 154 154 154 T F T <--- 26 520-539 90 90 90 151 151 151 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 92 92 92 153 153 153 F F F Max gen: 94, Max flush: 156 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 157 157 157 F F F 1 020-039 95 95 95 157 157 157 F F F 2 040-059 80 80 80 135 135 135 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 95 95 95 157 157 157 F F F 5 100-119 94 94 94 155 155 155 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 87 87 87 144 144 144 F F F 11 220-239 87 87 87 144 144 144 F F F 12 240-259 93 93 93 154 154 154 F F F 13 260-279 94 94 94 156 156 156 F F F 14 280-299 94 94 94 155 155 155 F F F 15 300-319 95 95 95 157 157 157 F F F 16 320-339 94 94 94 155 155 155 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 95 95 95 157 157 157 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 91 91 91 152 152 152 F F F 21 420-439 88 88 88 147 147 147 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 94 94 94 156 156 156 F F F 24 480-499 94 94 94 155 155 155 F F F 25 500-519 95 95 95 157 157 157 F F F 26 520-539 90 90 90 151 151 151 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 92 92 92 153 153 153 F F F Max gen: 95, Max flush: 157 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.934553924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.942867187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943154898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943179636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943188346Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943200145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Crucible a3fe2751-d8ff-42fe-b679-22ef0901f601 has session id: 46870ce8-43e5-4ade-ba06-67ce80388837","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943561341Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Upstairs opts: Upstairs UUID: a3fe2751-d8ff-42fe-b679-22ef0901f601, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.94358439Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Crucible stats registered with UUID: a3fe2751-d8ff-42fe-b679-22ef0901f601","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943597189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:45.943606498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943752919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943768318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943778337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943943076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943959665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"a3fe2751-d8ff-42fe-b679-22ef0901f601 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.943993763Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944003362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"0","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944013332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"1","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944023081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"2","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.94403274Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.94404287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944076997Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"ds_connection connected from Ok(127.0.0.1:37168)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944096176Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"ds_connection connected from Ok(127.0.0.1:37776)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944107205Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"ds_connection connected from Ok(127.0.0.1:51947)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.94419514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944871725Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"1","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944883784Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"0","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.944894594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"2","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945230222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945239821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[0]R flush_numbers[0..12]: [157, 157, 135, 143, 157, 155, 153, 152, 151, 153, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.9452595Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[0]R generation[0..12]: [95, 95, 80, 86, 95, 94, 92, 91, 90, 92, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945270239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945281268Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[1]R flush_numbers[0..12]: [157, 157, 135, 143, 157, 155, 153, 152, 151, 153, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945291947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[1]R generation[0..12]: [95, 95, 80, 86, 95, 94, 92, 91, 90, 92, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945302247Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945313156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[2]R flush_numbers[0..12]: [157, 157, 135, 143, 157, 155, 153, 152, 151, 153, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945323765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[2]R generation[0..12]: [95, 95, 80, 86, 95, 94, 92, 91, 90, 92, 87, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945334145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945347014Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945356363Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945366113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945375372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945384821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"":"downstairs","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945394561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.94540322Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"0","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945419319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"1","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945429318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"client":"2","":"downstairs","session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945439388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945448807Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"a3fe2751-d8ff-42fe-b679-22ef0901f601 is now active with session: 46870ce8-43e5-4ade-ba06-67ce80388837","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945458836Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945467556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"session_id":"46870ce8-43e5-4ade-ba06-67ce80388837"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945476605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Activated sub_volume a3fe2751-d8ff-42fe-b679-22ef0901f601","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945484585Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945492654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.945825022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"task":"crutest"} 01/30 Flush 02/30 Write block 31 len 32768 data: 6 6 5 4 3 3 5 5 03/30 Write block 99 len 36864 data: 11 12 13 13 13 11 10 8 9 04/30 Write block 208 len 8192 data: 11 12 05/30 Read block 457 len 16384 06/30 Read block 25 len 24576 07/30 Read block 364 len 16384 08/30 Read block 518 len 40960 09/30 Read block 182 len 28672 10/30 Write block 263 len 16384 data: 4 6 6 8 11/30 Flush 12/30 Write block 422 len 32768 data: 10 7 8 8 9 10 7 7 13/30 Read block 427 len 28672 14/30 Flush 15/30 Write block 33 len 32768 data: 6 5 4 4 6 6 4 3 16/30 Read block 245 len 28672 17/30 Write block 235 len 20480 data: 8 8 9 9 9 18/30 Write block 580 len 8192 data: 5 4 19/30 Write block 302 len 36864 data: 7 8 10 9 8 9 9 9 11 20/30 Write block 467 len 24576 data: 6 5 6 7 9 10 21/30 Write block 488 len 8192 data: 11 10 22/30 Read block 0 len 8192 23/30 Flush 24/30 Write block 418 len 36864 data: 7 7 7 8 11 8 9 9 10 25/30 Read block 576 len 8192 26/30 Write block 250 len 36864 data: 11 9 8 8 7 5 6 5 5 27/30 Read block 462 len 8192 28/30 Write block 261 len 8192 data: 7 5 29/30 Write block 529 len 32768 data: 6 7 7 6 6 5 7 8 30/30 Write block 279 len 40960 data: 4 3 5 6 6 7 9 9 8 11 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 24 27 Done 28 27 5 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:45.954693349Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:45.954901645Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1523,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 157 157 157 F F F 1 020-039 96 96 95 161 161 157 F F T <--- 2 040-059 96 96 80 161 161 135 F F F <--- 3 060-079 86 86 86 143 143 143 F F F 4 080-099 96 96 95 159 159 157 F F T <--- 5 100-119 96 96 94 159 159 155 F F T <--- 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 96 96 87 159 159 144 F F T <--- 11 220-239 96 96 87 161 161 144 F F F <--- 12 240-259 93 93 93 154 154 154 T T F <--- 13 260-279 96 96 94 159 159 156 T T F <--- 14 280-299 94 94 94 155 155 155 T T F <--- 15 300-319 96 96 95 161 161 157 F F F <--- 16 320-339 94 94 94 155 155 155 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 95 95 95 157 157 157 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 91 91 91 152 152 152 T T F <--- 21 420-439 96 96 88 160 160 147 T T F <--- 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 94 161 161 156 F F F <--- 24 480-499 96 96 94 161 161 155 F F F <--- 25 500-519 95 95 95 157 157 157 F F F 26 520-539 90 90 90 151 151 151 T T F <--- 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 96 96 92 161 161 153 F F F <--- Max gen: 96, Max flush: 161 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 157 157 157 F F F 1 020-039 96 96 96 161 161 161 F F F 2 040-059 96 96 96 161 161 161 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 96 96 96 159 159 159 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 96 96 96 159 159 159 F F F 11 220-239 96 96 96 161 161 161 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 97 97 97 162 162 162 F F F 14 280-299 97 97 97 162 162 162 F F F 15 300-319 96 96 96 161 161 161 F F F 16 320-339 94 94 94 155 155 155 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 95 95 95 157 157 157 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 97 97 97 162 162 162 F F F 21 420-439 97 97 97 162 162 162 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 96 96 96 161 161 161 F F F 25 500-519 95 95 95 157 157 157 F F F 26 520-539 97 97 97 162 162 162 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 91 91 91 152 152 152 F F F 29 580-599 96 96 96 161 161 161 F F F Max gen: 97, Max flush: 162 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.219854497Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228280213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228573373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228600702Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228609361Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.22862101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"Crucible dd5f08ab-5f66-40e0-9c97-0cc71d280155 has session id: 003a5dde-b565-44ce-b9ad-75137f8258a0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228921011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Upstairs opts: Upstairs UUID: dd5f08ab-5f66-40e0-9c97-0cc71d280155, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228962038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Crucible stats registered with UUID: dd5f08ab-5f66-40e0-9c97-0cc71d280155","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.228972877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:50.228982297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229144396Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229159405Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229169644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229308175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229324834Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"dd5f08ab-5f66-40e0-9c97-0cc71d280155 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229348202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229357442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"0","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229367231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"1","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229376831Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"2","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.22938641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229396429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229406439Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44858)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229448156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34363)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229458895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63758)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.229560838Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230126931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"0","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.23015362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"1","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.23029073Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"2","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230471169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230481658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[0]R flush_numbers[0..12]: [157, 161, 161, 143, 159, 159, 153, 152, 151, 153, 159, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230588331Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[0]R generation[0..12]: [95, 96, 96, 86, 96, 96, 92, 91, 90, 92, 96, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.23059957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230610799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[1]R flush_numbers[0..12]: [157, 161, 161, 143, 159, 159, 153, 152, 151, 153, 159, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230621729Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[1]R generation[0..12]: [95, 96, 96, 86, 96, 96, 92, 91, 90, 92, 96, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230632318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230643407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[2]R flush_numbers[0..12]: [157, 161, 161, 143, 159, 159, 153, 152, 151, 153, 159, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230654267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[2]R generation[0..12]: [95, 96, 96, 86, 96, 96, 92, 91, 90, 92, 96, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230664766Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230675855Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230685255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230695184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230704643Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230714203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"":"downstairs","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230725932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230734771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"0","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230744401Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"1","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.23075394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"client":"2","":"downstairs","session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230763709Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230772999Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"dd5f08ab-5f66-40e0-9c97-0cc71d280155 is now active with session: 003a5dde-b565-44ce-b9ad-75137f8258a0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230783078Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230791728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"session_id":"003a5dde-b565-44ce-b9ad-75137f8258a0"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230800827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"Activated sub_volume dd5f08ab-5f66-40e0-9c97-0cc71d280155","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230808906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.230822456Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.231082538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"task":"crutest"} 01/30 Write block 548 len 8192 data: 10 10 02/30 Read block 8 len 4096 03/30 Read block 593 len 28672 04/30 Write block 432 len 28672 data: 8 6 7 8 8 8 9 05/30 Flush 06/30 Write block 211 len 24576 data: 16 16 15 16 17 15 07/30 Write block 34 len 16384 data: 6 5 5 7 08/30 Write block 574 len 16384 data: 4 5 7 7 09/30 Read block 530 len 32768 10/30 Read block 428 len 20480 11/30 Read block 463 len 36864 12/30 Read block 342 len 24576 13/30 Flush 14/30 Write block 366 len 24576 data: 12 11 11 9 8 6 15/30 Read block 308 len 32768 16/30 Read block 331 len 16384 17/30 Read block 261 len 8192 18/30 Write block 273 len 40960 data: 7 8 6 6 3 3 5 4 6 7 19/30 Write block 22 len 32768 data: 10 9 7 6 6 5 6 6 20/30 Write block 281 len 16384 data: 7 8 7 8 21/30 Read block 161 len 32768 22/30 Read block 98 len 20480 23/30 Read block 12 len 12288 24/30 Read block 27 len 24576 25/30 Write block 51 len 24576 data: 8 7 8 7 5 6 26/30 Read block 0 len 20480 27/30 Read block 597 len 8192 28/30 Write block 326 len 20480 data: 8 8 9 10 10 29/30 Read block 248 len 16384 30/30 Read block 50 len 8192 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) None Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:50.239769477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:50.239936796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1531,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 157 157 157 F F F 1 020-039 98 96 98 164 161 164 T F T <--- 2 040-059 96 96 96 161 161 161 T F T <--- 3 060-079 86 86 86 143 143 143 F F F 4 080-099 96 96 96 159 159 159 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 98 96 98 164 159 164 F F F <--- 11 220-239 96 96 96 161 161 161 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 97 97 97 162 162 162 T F T <--- 14 280-299 97 97 97 162 162 162 T F T <--- 15 300-319 96 96 96 161 161 161 F F F 16 320-339 94 94 94 155 155 155 T F T <--- 17 340-359 88 88 88 147 147 147 F F F 18 360-379 95 95 95 157 157 157 T F T <--- 19 380-399 95 95 95 157 157 157 F F F 20 400-419 97 97 97 162 162 162 F F F 21 420-439 98 97 98 163 162 163 F F F <--- 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 96 96 96 161 161 161 F F F 25 500-519 95 95 95 157 157 157 F F F 26 520-539 97 97 97 162 162 162 F F F 27 540-559 98 88 98 163 147 163 F T F <--- 28 560-579 98 91 98 164 152 164 F F F <--- 29 580-599 96 96 96 161 161 161 F F F Max gen: 98, Max flush: 164 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 157 157 157 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 86 86 86 143 143 143 F F F 4 080-099 96 96 96 159 159 159 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 98 98 98 164 164 164 F F F 11 220-239 96 96 96 161 161 161 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 99 99 99 165 165 165 F F F 14 280-299 99 99 99 165 165 165 F F F 15 300-319 96 96 96 161 161 161 F F F 16 320-339 99 99 99 165 165 165 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 99 99 99 165 165 165 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 97 97 97 162 162 162 F F F 21 420-439 98 98 98 163 163 163 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 96 96 96 161 161 161 F F F 25 500-519 95 95 95 157 157 157 F F F 26 520-539 97 97 97 162 162 162 F F F 27 540-559 98 98 98 163 163 163 F F F 28 560-579 98 98 98 164 164 164 F F F 29 580-599 96 96 96 161 161 161 F F F Max gen: 99, Max flush: 165 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.48599815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:54.493242694Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.49329976Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.500351356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.500605609Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.500633887Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.500642497Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.500654346Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Crucible 9d89762c-43ec-410f-8206-aa0dd8663aed has session id: b2fa3ad3-2ba7-4a48-acfd-aad42dd976df","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501030941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Upstairs opts: Upstairs UUID: 9d89762c-43ec-410f-8206-aa0dd8663aed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.50105568Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Crucible stats registered with UUID: 9d89762c-43ec-410f-8206-aa0dd8663aed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501066019Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:54.501075138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501216159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501231658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501241728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501397267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501421056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"9d89762c-43ec-410f-8206-aa0dd8663aed active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501437355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501446394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"0","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501458253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"1","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501468033Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"2","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501477712Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501487661Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501497831Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"ds_connection connected from Ok(127.0.0.1:43380)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501537618Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"ds_connection connected from Ok(127.0.0.1:52030)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501563286Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"ds_connection connected from Ok(127.0.0.1:56352)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.501574996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502124649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"1","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502174756Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"2","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502188905Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"0","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502444658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502466007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[0]R flush_numbers[0..12]: [157, 165, 165, 143, 159, 159, 153, 152, 151, 153, 164, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502476366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[0]R generation[0..12]: [95, 99, 99, 86, 96, 96, 92, 91, 90, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502487396Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502501325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[1]R flush_numbers[0..12]: [157, 165, 165, 143, 159, 159, 153, 152, 151, 153, 164, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502512104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[1]R generation[0..12]: [95, 99, 99, 86, 96, 96, 92, 91, 90, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502522653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502533733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[2]R flush_numbers[0..12]: [157, 165, 165, 143, 159, 159, 153, 152, 151, 153, 164, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502544332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[2]R generation[0..12]: [95, 99, 99, 86, 96, 96, 92, 91, 90, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502554651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.50256571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.50257542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502585269Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502594689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502604108Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"":"downstairs","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502614037Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502622757Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"0","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502632636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"1","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502642165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"client":"2","":"downstairs","session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502653415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502662644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"9d89762c-43ec-410f-8206-aa0dd8663aed is now active with session: b2fa3ad3-2ba7-4a48-acfd-aad42dd976df","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502672613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502681393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"session_id":"b2fa3ad3-2ba7-4a48-acfd-aad42dd976df"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502690602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Activated sub_volume 9d89762c-43ec-410f-8206-aa0dd8663aed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502698762Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.502706941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.50302177Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} 01/30 Write block 147 len 20480 data: 12 12 13 11 12 02/30 Write block 76 len 32768 data: 10 7 5 5 5 5 4 4 03/30 Read block 514 len 36864 04/30 Write block 216 len 16384 data: 16 12 11 8 05/30 Write block 587 len 32768 data: 6 8 7 8 8 8 7 9 06/30 Write block 366 len 28672 data: 13 12 12 10 9 7 6 07/30 Read block 355 len 40960 08/30 Write block 426 len 4096 data: 11 09/30 Read block 117 len 4096 10/30 Read block 349 len 12288 11/30 Write block 523 len 40960 data: 5 4 5 6 6 5 7 8 8 7 12/30 Read block 145 len 28672 13/30 Read block 569 len 20480 14/30 Read block 8 len 28672 15/30 Write block 297 len 28672 data: 6 6 7 8 7 8 9 16/30 Read block 344 len 32768 17/30 Write block 4 len 4096 data: 7 18/30 Write block 500 len 24576 data: 6 7 5 7 8 8 19/30 Read block 5 len 36864 20/30 Read block 439 len 40960 21/30 Write block 268 len 24576 data: 9 8 7 6 7 8 22/30 Read block 139 len 16384 23/30 Write block 412 len 16384 data: 8 9 8 9 24/30 Read block 535 len 24576 25/30 Read block 466 len 28672 26/30 Read block 295 len 12288 27/30 Read block 476 len 36864 28/30 Read block 60 len 16384 29/30 Write block 283 len 12288 data: 8 9 10 30/30 Read block 21 len 20480 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:54.511633574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:54.511820542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1539,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 157 157 157 F T T <--- 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 86 86 86 143 143 143 T T T 4 080-099 96 96 96 159 159 159 T T T 5 100-119 96 96 96 159 159 159 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 91 91 91 152 152 152 T T T 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 98 98 98 164 164 164 F T T <--- 11 220-239 96 96 96 161 161 161 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 99 99 99 165 165 165 F T T <--- 14 280-299 99 99 99 165 165 165 F T T <--- 15 300-319 96 96 96 161 161 161 F T T <--- 16 320-339 99 99 99 165 165 165 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 99 99 99 165 165 165 F T T <--- 19 380-399 95 95 95 157 157 157 F F F 20 400-419 97 97 97 162 162 162 F T T <--- 21 420-439 98 98 98 163 163 163 F T T <--- 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 96 96 96 161 161 161 F F F 25 500-519 95 95 95 157 157 157 F T T <--- 26 520-539 97 97 97 162 162 162 F T T <--- 27 540-559 98 98 98 163 163 163 F F F 28 560-579 98 98 98 164 164 164 F F F 29 580-599 96 96 96 161 161 161 F T T <--- Max gen: 99, Max flush: 165 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 96 96 96 159 159 159 F F F 6 120-139 92 92 92 153 153 153 F F F 7 140-159 100 100 100 166 166 166 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 100 100 100 166 166 166 F F F 11 220-239 96 96 96 161 161 161 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 100 100 100 166 166 166 F F F 14 280-299 100 100 100 166 166 166 F F F 15 300-319 100 100 100 166 166 166 F F F 16 320-339 99 99 99 165 165 165 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 100 100 100 166 166 166 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 100 100 100 166 166 166 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 96 96 96 161 161 161 F F F 25 500-519 100 100 100 166 166 166 F F F 26 520-539 100 100 100 166 166 166 F F F 27 540-559 98 98 98 163 163 163 F F F 28 560-579 98 98 98 164 164 164 F F F 29 580-599 100 100 100 166 166 166 F F F Max gen: 100, Max flush: 166 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.780374417Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:58.78763339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.787683597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.794806148Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795063341Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795098189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795107638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795119808Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Crucible e24a72b3-500c-4f1f-83ff-63f5d63438e2 has session id: 9b03a3f2-67a8-40d8-8846-aed98d36e66c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795425197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Upstairs opts: Upstairs UUID: e24a72b3-500c-4f1f-83ff-63f5d63438e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795449376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Crucible stats registered with UUID: e24a72b3-500c-4f1f-83ff-63f5d63438e2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795459275Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:58.795468375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795591566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795606476Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795616635Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795787374Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795811092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"e24a72b3-500c-4f1f-83ff-63f5d63438e2 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.79583472Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.79584431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"0","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795854039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"1","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795863799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"2","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795873158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795883307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48139)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795921315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795931744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33320)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.795942083Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52799)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.79599865Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796518106Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"1","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796589671Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"2","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796615649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"0","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796839864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796863913Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[0]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 159, 153, 166, 151, 153, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796940308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[0]R generation[0..12]: [100, 99, 99, 100, 100, 96, 92, 100, 90, 92, 100, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796952187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796963356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[1]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 159, 153, 166, 151, 153, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796974216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[1]R generation[0..12]: [100, 99, 99, 100, 100, 96, 92, 100, 90, 92, 100, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796984775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.796998604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[2]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 159, 153, 166, 151, 153, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797009413Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[2]R generation[0..12]: [100, 99, 99, 100, 100, 96, 92, 100, 90, 92, 100, 96]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797020003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797030952Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Max found gen is 101","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797040461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Generation requested: 102 >= found:101","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797050241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.79705962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797069249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"":"downstairs","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797079119Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797087978Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"0","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797097747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"1","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797107417Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"client":"2","":"downstairs","session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797117326Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797126496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"e24a72b3-500c-4f1f-83ff-63f5d63438e2 is now active with session: 9b03a3f2-67a8-40d8-8846-aed98d36e66c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797136585Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797147094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"session_id":"9b03a3f2-67a8-40d8-8846-aed98d36e66c"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797156234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Activated sub_volume e24a72b3-500c-4f1f-83ff-63f5d63438e2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797164213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797172343Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.797463013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} 01/30 Flush 02/30 Write block 215 len 28672 data: 18 17 13 12 9 7 6 03/30 Write block 495 len 16384 data: 10 10 11 10 04/30 Read block 59 len 12288 05/30 Read block 311 len 40960 06/30 Flush 07/30 Write block 569 len 4096 data: 7 08/30 Read block 160 len 32768 09/30 Write block 136 len 16384 data: 10 11 8 8 10/30 Flush 11/30 Write block 402 len 28672 data: 9 10 9 9 7 7 7 12/30 Read block 304 len 4096 13/30 Read block 496 len 12288 14/30 Read block 169 len 24576 15/30 Write block 366 len 32768 data: 14 13 13 11 10 8 7 5 16/30 Write block 408 len 28672 data: 8 7 8 9 9 10 9 17/30 Write block 310 len 28672 data: 12 10 9 7 7 5 7 18/30 Write block 588 len 40960 data: 9 8 9 9 9 8 10 8 7 7 19/30 Read block 408 len 8192 20/30 Write block 363 len 12288 data: 12 13 11 21/30 Read block 80 len 24576 22/30 Write block 206 len 8192 data: 10 9 23/30 Flush 24/30 Write block 221 len 40960 data: 7 6 5 4 4 5 7 7 8 6 25/30 Read block 297 len 24576 26/30 Write block 537 len 24576 data: 7 8 8 9 9 9 27/30 Write block 112 len 28672 data: 8 6 6 5 7 8 9 28/30 Write block 230 len 4096 data: 7 29/30 Read block 67 len 40960 30/30 Read block 404 len 40960 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:31:58.805909278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:31:58.806091306Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1547,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 96 96 96 159 159 159 F T T <--- 6 120-139 92 102 102 153 169 169 F F F <--- 7 140-159 100 100 100 166 166 166 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 100 102 102 166 170 170 F F F <--- 11 220-239 96 102 102 161 168 168 F T T <--- 12 240-259 97 97 97 162 162 162 F F F 13 260-279 100 100 100 166 166 166 F F F 14 280-299 100 100 100 166 166 166 F F F 15 300-319 100 102 102 166 170 170 F F F <--- 16 320-339 99 99 99 165 165 165 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 100 102 102 166 170 170 F F F <--- 19 380-399 95 95 95 157 157 157 F F F 20 400-419 100 102 102 166 170 170 F F F <--- 21 420-439 100 100 100 166 166 166 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 96 102 102 161 168 168 F F F <--- 25 500-519 100 100 100 166 166 166 F F F 26 520-539 100 100 100 166 166 166 F T T <--- 27 540-559 98 98 98 163 163 163 F T T <--- 28 560-579 98 102 102 164 169 169 F F F <--- 29 580-599 100 102 102 166 170 170 F F F <--- Max gen: 102, Max flush: 170 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 103 103 103 171 171 171 F F F 6 120-139 102 102 102 169 169 169 F F F 7 140-159 100 100 100 166 166 166 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 92 92 92 153 153 153 F F F 10 200-219 102 102 102 170 170 170 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 100 100 100 166 166 166 F F F 14 280-299 100 100 100 166 166 166 F F F 15 300-319 102 102 102 170 170 170 F F F 16 320-339 99 99 99 165 165 165 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 102 102 102 170 170 170 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 102 102 102 168 168 168 F F F 25 500-519 100 100 100 166 166 166 F F F 26 520-539 103 103 103 171 171 171 F F F 27 540-559 103 103 103 171 171 171 F F F 28 560-579 102 102 102 169 169 169 F F F 29 580-599 102 102 102 170 170 170 F F F Max gen: 103, Max flush: 171 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.062802809Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.07116407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.071442081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.07146745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.071476249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.071488058Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Crucible 3c0e3523-8726-4400-929a-5323e1381f64 has session id: 60080d41-c5a6-4c71-b5d9-bfc202874d0d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.071877553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Upstairs opts: Upstairs UUID: 3c0e3523-8726-4400-929a-5323e1381f64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.071936279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Crucible stats registered with UUID: 3c0e3523-8726-4400-929a-5323e1381f64","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.071961647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:03.071972296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072084439Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072104718Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072114957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072268967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072284566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"3c0e3523-8726-4400-929a-5323e1381f64 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072308094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072317674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"0","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072327373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"1","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072337082Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"2","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072346592Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.07237984Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072403788Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46493)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072446255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54264)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072559048Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52535)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.072584086Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073023167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"0","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073042796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"2","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073121931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"1","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073336187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073353556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[0]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 171, 169, 166, 151, 153, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073406732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[0]R generation[0..12]: [100, 99, 99, 100, 100, 103, 102, 100, 90, 92, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073418111Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073429291Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[1]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 171, 169, 166, 151, 153, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.07343997Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[1]R generation[0..12]: [100, 99, 99, 100, 100, 103, 102, 100, 90, 92, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073450759Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073461798Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[2]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 171, 169, 166, 151, 153, 170, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073472548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[2]R generation[0..12]: [100, 99, 99, 100, 100, 103, 102, 100, 90, 92, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073483187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073494186Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073503636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073515155Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073524534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073533994Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"":"downstairs","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073543743Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073552522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"0","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073562272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"1","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073571941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"client":"2","":"downstairs","session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073581721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.07359091Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"3c0e3523-8726-4400-929a-5323e1381f64 is now active with session: 60080d41-c5a6-4c71-b5d9-bfc202874d0d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073601059Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073609779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"session_id":"60080d41-c5a6-4c71-b5d9-bfc202874d0d"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073618898Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Activated sub_volume 3c0e3523-8726-4400-929a-5323e1381f64","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.073626998Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"timeDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"2026-09-18T03:32:03.073635197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.07389013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"task":"crutest"} 01/30 Write block 206 len 12288 data: 11 10 12 02/30 Read block 201 len 8192 03/30 Write block 330 len 16384 data: 11 9 10 9 04/30 Read block 496 len 24576 05/30 Write block 279 len 20480 data: 6 5 8 9 9 06/30 Read block 371 len 24576 07/30 Read block 2 len 12288 08/30 Read block 189 len 36864 09/30 Flush 10/30 Write block 516 len 40960 data: 11 10 10 8 8 5 4 6 5 6 11/30 Read block 374 len 12288 12/30 Read block 540 len 40960 13/30 Write block 587 len 12288 data: 7 10 9 14/30 Write block 361 len 4096 data: 9 15/30 Write block 532 len 28672 data: 8 7 6 8 9 8 9 16/30 Read block 588 len 4096 17/30 Read block 334 len 8192 18/30 Read block 115 len 16384 19/30 Read block 161 len 8192 20/30 Write block 306 len 20480 data: 9 10 10 10 13 21/30 Read block 16 len 20480 22/30 Flush 23/30 Write block 187 len 24576 data: 9 9 8 8 4 6 24/30 Write block 537 len 36864 data: 9 10 9 10 10 10 10 10 11 25/30 Write block 211 len 8192 data: 17 17 26/30 Read block 441 len 8192 27/30 Read block 530 len 24576 28/30 Write block 208 len 40960 data: 13 13 12 18 18 16 17 19 18 14 29/30 Write block 170 len 24576 data: 7 8 5 7 8 7 30/30 Write block 176 len 20480 data: 7 9 8 9 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 26 3 32 Done 27 4 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:03.081765652Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:03.081940541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1555,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 103 103 103 171 171 171 F F F 6 120-139 102 102 102 169 169 169 F F F 7 140-159 100 100 100 166 166 166 F F F 8 160-179 90 90 90 151 151 151 T F T <--- 9 180-199 92 92 92 153 153 153 T F T <--- 10 200-219 104 102 104 172 170 172 T T T <--- 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 104 100 104 172 166 172 F F F <--- 14 280-299 104 100 104 172 166 172 F F F <--- 15 300-319 104 102 104 173 170 173 F F F <--- 16 320-339 104 99 104 172 165 172 F T F <--- 17 340-359 88 88 88 147 147 147 F F F 18 360-379 104 102 104 173 170 173 F F F <--- 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 102 102 102 168 168 168 F F F 25 500-519 104 100 104 173 166 173 F F F <--- 26 520-539 104 103 104 173 171 173 T F T <--- 27 540-559 103 103 103 171 171 171 T F T <--- 28 560-579 102 102 102 169 169 169 F F F 29 580-599 104 102 104 173 170 173 F F F <--- Max gen: 104, Max flush: 173 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 99 99 99 165 165 165 F F F 2 040-059 99 99 99 165 165 165 F F F 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 103 103 103 171 171 171 F F F 6 120-139 102 102 102 169 169 169 F F F 7 140-159 100 100 100 166 166 166 F F F 8 160-179 105 105 105 174 174 174 F F F 9 180-199 105 105 105 174 174 174 F F F 10 200-219 105 105 105 174 174 174 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 104 104 104 172 172 172 F F F 14 280-299 104 104 104 172 172 172 F F F 15 300-319 104 104 104 173 173 173 F F F 16 320-339 104 104 104 172 172 172 F F F 17 340-359 88 88 88 147 147 147 F F F 18 360-379 104 104 104 173 173 173 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 87 87 87 144 144 144 F F F 23 460-479 96 96 96 161 161 161 F F F 24 480-499 102 102 102 168 168 168 F F F 25 500-519 104 104 104 173 173 173 F F F 26 520-539 105 105 105 174 174 174 F F F 27 540-559 105 105 105 174 174 174 F F F 28 560-579 102 102 102 169 169 169 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 105, Max flush: 174 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.345125339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.353328819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.353601491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.353626329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.353634979Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.353646648Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Crucible ab9fea02-fa6f-4e7a-a56b-3a46605b4a07 has session id: 25c33232-c177-4123-b2be-012986bfcb2c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.353995335Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Upstairs opts: Upstairs UUID: ab9fea02-fa6f-4e7a-a56b-3a46605b4a07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354024453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Crucible stats registered with UUID: ab9fea02-fa6f-4e7a-a56b-3a46605b4a07","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354035133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:07.354044232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354182253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354197392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354207491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354386319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354427887Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"ab9fea02-fa6f-4e7a-a56b-3a46605b4a07 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354436636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354447625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"0","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354457445Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"1","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354467114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"2","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354476674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354486603Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354496442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42808)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354566158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33344)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354592176Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36439)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.354609465Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355256262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"0","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.35529091Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"2","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355303099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"1","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355578661Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.35558937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[0]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 171, 169, 166, 174, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355615619Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[0]R generation[0..12]: [100, 99, 99, 100, 100, 103, 102, 100, 105, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355629538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355640777Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[1]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 171, 169, 166, 174, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355651446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[1]R generation[0..12]: [100, 99, 99, 100, 100, 103, 102, 100, 105, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355661916Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355672845Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[2]R flush_numbers[0..12]: [166, 165, 165, 166, 166, 171, 169, 166, 174, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355683544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[2]R generation[0..12]: [100, 99, 99, 100, 100, 103, 102, 100, 105, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355694023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355705113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355714602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355724391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355733691Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.35574318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"":"downstairs","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.35575287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355761819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"0","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355780288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"1","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355789967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"client":"2","":"downstairs","session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355799857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355809256Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"ab9fea02-fa6f-4e7a-a56b-3a46605b4a07 is now active with session: 25c33232-c177-4123-b2be-012986bfcb2c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355819285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355828015Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"session_id":"25c33232-c177-4123-b2be-012986bfcb2c"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355837194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Activated sub_volume ab9fea02-fa6f-4e7a-a56b-3a46605b4a07","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355845344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.355853583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.356165662Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"task":"crutest"} 01/30 Write block 345 len 12288 data: 10 11 11 02/30 Write block 137 len 16384 data: 12 9 9 7 03/30 Read block 301 len 40960 04/30 Read block 234 len 32768 05/30 Read block 413 len 32768 06/30 Read block 122 len 8192 07/30 Flush 08/30 Write block 333 len 36864 data: 10 6 7 9 8 9 8 9 11 09/30 Read block 179 len 24576 10/30 Flush 11/30 Write block 546 len 40960 data: 8 11 11 11 10 8 7 8 8 10 12/30 Write block 111 len 20480 data: 8 9 7 7 6 13/30 Read block 322 len 40960 14/30 Read block 204 len 28672 15/30 Read block 31 len 40960 16/30 Read block 86 len 24576 17/30 Read block 83 len 36864 18/30 Write block 37 len 28672 data: 8 7 5 4 4 6 6 19/30 Read block 282 len 4096 20/30 Read block 352 len 24576 21/30 Write block 452 len 36864 data: 8 9 8 7 8 6 6 8 8 22/30 Read block 488 len 24576 23/30 Write block 40 len 20480 data: 5 5 7 7 7 24/30 Write block 352 len 32768 data: 8 7 8 7 7 9 7 10 25/30 Write block 108 len 36864 data: 9 8 8 9 10 8 8 7 8 26/30 Read block 97 len 40960 27/30 Read block 474 len 24576 28/30 Write block 156 len 28672 data: 10 12 11 13 10 10 9 29/30 Read block 178 len 8192 30/30 Write block 299 len 28672 data: 8 9 8 9 10 11 10 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:07.365504548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:07.365678197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1563,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 99 99 99 165 165 165 T F T <--- 2 040-059 99 99 99 165 165 165 T F T <--- 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 103 103 103 171 171 171 T F T <--- 6 120-139 106 102 106 175 169 175 F T F <--- 7 140-159 106 100 106 175 166 175 T T T <--- 8 160-179 105 105 105 174 174 174 T F T <--- 9 180-199 105 105 105 174 174 174 F F F 10 200-219 105 105 105 174 174 174 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 104 104 104 172 172 172 F F F 14 280-299 104 104 104 172 172 172 T F T <--- 15 300-319 104 104 104 173 173 173 T F T <--- 16 320-339 106 104 106 176 172 176 F F F <--- 17 340-359 106 88 106 176 147 176 T T T <--- 18 360-379 104 104 104 173 173 173 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 87 87 87 144 144 144 T F T <--- 23 460-479 96 96 96 161 161 161 T F T <--- 24 480-499 102 102 102 168 168 168 F F F 25 500-519 104 104 104 173 173 173 F F F 26 520-539 105 105 105 174 174 174 F F F 27 540-559 105 105 105 174 174 174 T F T <--- 28 560-579 102 102 102 169 169 169 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 106, Max flush: 176 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 107 107 107 177 177 177 F F F 3 060-079 100 100 100 166 166 166 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 106 106 106 175 175 175 F F F 7 140-159 107 107 107 177 177 177 F F F 8 160-179 107 107 107 177 177 177 F F F 9 180-199 105 105 105 174 174 174 F F F 10 200-219 105 105 105 174 174 174 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 104 104 104 172 172 172 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 106 106 106 176 176 176 F F F 17 340-359 107 107 107 177 177 177 F F F 18 360-379 104 104 104 173 173 173 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 107 107 107 177 177 177 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 102 102 102 168 168 168 F F F 25 500-519 104 104 104 173 173 173 F F F 26 520-539 105 105 105 174 174 174 F F F 27 540-559 107 107 107 177 177 177 F F F 28 560-579 102 102 102 169 169 169 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 107, Max flush: 177 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.627137899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.635499309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.63579476Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.635819418Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.635828217Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.635840067Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Crucible ed6c7ee8-c428-4b51-9b69-8f8c2291d0e7 has session id: 7e557d21-dc62-45ff-8539-887b09996991","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636184024Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Upstairs opts: Upstairs UUID: ed6c7ee8-c428-4b51-9b69-8f8c2291d0e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636212962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Crucible stats registered with UUID: ed6c7ee8-c428-4b51-9b69-8f8c2291d0e7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636223771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:11.636232961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636383361Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636412939Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636423978Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636538181Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.63655537Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"ed6c7ee8-c428-4b51-9b69-8f8c2291d0e7 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636588557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636598197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"0","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636607886Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"1","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636617696Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"2","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636627495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636637574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636650263Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"ds_connection connected from Ok(127.0.0.1:59335)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636711319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"ds_connection connected from Ok(127.0.0.1:44334)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636733358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"ds_connection connected from Ok(127.0.0.1:61659)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.636842531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637349297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"0","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637375586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"2","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637387785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"1","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637659737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637669446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[0]R flush_numbers[0..12]: [166, 177, 177, 166, 166, 177, 175, 177, 177, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637704834Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[0]R generation[0..12]: [100, 107, 107, 100, 100, 107, 106, 107, 107, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637716243Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637727613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[1]R flush_numbers[0..12]: [166, 177, 177, 166, 166, 177, 175, 177, 177, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637738482Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[1]R generation[0..12]: [100, 107, 107, 100, 100, 107, 106, 107, 107, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637749461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.63776336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[2]R flush_numbers[0..12]: [166, 177, 177, 166, 166, 177, 175, 177, 177, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637774479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[2]R generation[0..12]: [100, 107, 107, 100, 100, 107, 106, 107, 107, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637785299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637796358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637805907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637815767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637825246Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637834815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"":"downstairs","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637844645Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637853514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"0","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637863224Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"1","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637872893Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"client":"2","":"downstairs","session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637882822Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637891942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"ed6c7ee8-c428-4b51-9b69-8f8c2291d0e7 is now active with session: 7e557d21-dc62-45ff-8539-887b09996991","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637902061Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.63791269Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"session_id":"7e557d21-dc62-45ff-8539-887b09996991"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.63792196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Activated sub_volume ed6c7ee8-c428-4b51-9b69-8f8c2291d0e7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637930189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.637938449Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.63821837Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"task":"crutest"} 01/30 Write block 574 len 20480 data: 5 6 8 8 6 02/30 Read block 6 len 40960 03/30 Write block 149 len 16384 data: 14 12 13 12 04/30 Read block 432 len 28672 05/30 Read block 513 len 4096 06/30 Read block 438 len 8192 07/30 Write block 71 len 32768 data: 11 12 11 11 11 11 8 6 08/30 Read block 348 len 36864 09/30 Read block 164 len 36864 10/30 Read block 255 len 20480 11/30 Write block 546 len 24576 data: 9 12 12 12 11 9 12/30 Flush 13/30 Write block 144 len 32768 data: 9 9 10 13 13 15 13 14 14/30 Read block 521 len 8192 15/30 Read block 22 len 32768 16/30 Write block 120 len 36864 data: 11 9 10 11 11 10 10 10 10 17/30 Read block 126 len 32768 18/30 Write block 75 len 12288 data: 12 12 9 19/30 Write block 129 len 20480 data: 10 8 11 10 10 20/30 Write block 45 len 20480 data: 6 6 7 7 6 21/30 Read block 286 len 4096 22/30 Write block 270 len 20480 data: 8 7 8 9 9 23/30 Write block 341 len 32768 data: 12 11 10 10 11 12 12 10 24/30 Flush 25/30 Write block 515 len 24576 data: 11 12 11 11 9 9 26/30 Write block 345 len 36864 data: 12 13 13 11 9 9 8 9 8 27/30 Read block 196 len 4096 28/30 Read block 207 len 24576 29/30 Read block 190 len 32768 30/30 Write block 10 len 16384 data: 9 8 8 5 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 10 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:11.647158592Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:11.64735016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1571,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 166 166 166 T F T <--- 1 020-039 107 107 107 177 177 177 F F F 2 040-059 108 107 108 179 177 179 F F F <--- 3 060-079 108 100 108 179 166 179 F F F <--- 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 106 108 179 175 179 F F F <--- 7 140-159 108 107 108 179 177 179 F F F <--- 8 160-179 107 107 107 177 177 177 F F F 9 180-199 105 105 105 174 174 174 F F F 10 200-219 105 105 105 174 174 174 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 108 104 108 179 172 179 F F F <--- 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 106 106 106 176 176 176 F F F 17 340-359 108 107 108 179 177 179 T F T <--- 18 360-379 104 104 104 173 173 173 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 107 107 107 177 177 177 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 102 102 102 168 168 168 F F F 25 500-519 104 104 104 173 173 173 T F T <--- 26 520-539 105 105 105 174 174 174 T F T <--- 27 540-559 108 107 108 178 177 178 F F F <--- 28 560-579 108 102 108 178 169 178 F F F <--- 29 580-599 104 104 104 173 173 173 F F F Max gen: 108, Max flush: 179 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 108 108 108 179 179 179 F F F 3 060-079 108 108 108 179 179 179 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 108 108 108 179 179 179 F F F 8 160-179 107 107 107 177 177 177 F F F 9 180-199 105 105 105 174 174 174 F F F 10 200-219 105 105 105 174 174 174 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 108 108 108 179 179 179 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 106 106 106 176 176 176 F F F 17 340-359 109 109 109 180 180 180 F F F 18 360-379 104 104 104 173 173 173 F F F 19 380-399 95 95 95 157 157 157 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 107 107 107 177 177 177 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 102 102 102 168 168 168 F F F 25 500-519 109 109 109 180 180 180 F F F 26 520-539 109 109 109 180 180 180 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 108 108 108 178 178 178 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 109, Max flush: 180 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.917693047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.925929665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926228985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926255723Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926264213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926276082Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Crucible 59a60568-66f2-4a8a-b4cc-f9587713b5bb has session id: 31b7c74f-9cf7-42b6-b8dd-231205e9b6d7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926601221Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Upstairs opts: Upstairs UUID: 59a60568-66f2-4a8a-b4cc-f9587713b5bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926622929Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Crucible stats registered with UUID: 59a60568-66f2-4a8a-b4cc-f9587713b5bb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926632619Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:15.926708164Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926876043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.926905571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.92691653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927052461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.92706502Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"59a60568-66f2-4a8a-b4cc-f9587713b5bb active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927088629Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927098248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"0","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927107897Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"1","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927122826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"2","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927132296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927142395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927152345Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41236)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927192732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56179)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.92722783Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57797)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927239249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927869877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"1","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927898625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"0","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.927910565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"2","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928199796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928220054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[0]R flush_numbers[0..12]: [180, 177, 179, 179, 166, 177, 179, 179, 177, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928269051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 108, 100, 107, 108, 108, 107, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.92828047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.92829188Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[1]R flush_numbers[0..12]: [180, 177, 179, 179, 166, 177, 179, 179, 177, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928302919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 108, 100, 107, 108, 108, 107, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928313658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928324777Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[2]R flush_numbers[0..12]: [180, 177, 179, 179, 166, 177, 179, 179, 177, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928335557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 108, 100, 107, 108, 108, 107, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928346166Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928357085Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928369504Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928379304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928388693Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928398163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"":"downstairs","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928407982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928416811Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"0","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928426601Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"1","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.92843634Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"client":"2","":"downstairs","session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928446179Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928455399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"59a60568-66f2-4a8a-b4cc-f9587713b5bb is now active with session: 31b7c74f-9cf7-42b6-b8dd-231205e9b6d7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928465408Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928474188Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"session_id":"31b7c74f-9cf7-42b6-b8dd-231205e9b6d7"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928483317Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Activated sub_volume 59a60568-66f2-4a8a-b4cc-f9587713b5bb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928491336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928499496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.928809116Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"task":"crutest"} 01/30 Flush 02/30 Write block 372 len 32768 data: 8 6 6 7 5 5 5 4 03/30 Read block 82 len 24576 04/30 Flush 05/30 Write block 510 len 4096 data: 8 06/30 Read block 564 len 12288 07/30 Write block 204 len 20480 data: 11 11 12 11 14 08/30 Read block 192 len 40960 09/30 Write block 392 len 12288 data: 11 11 10 10/30 Read block 117 len 24576 11/30 Read block 550 len 12288 12/30 Write block 154 len 32768 data: 11 8 11 13 12 14 11 11 13/30 Read block 157 len 16384 14/30 Write block 149 len 12288 data: 16 14 15 15/30 Read block 273 len 4096 16/30 Write block 200 len 20480 data: 10 8 9 11 12 17/30 Read block 569 len 20480 18/30 Write block 442 len 32768 data: 8 7 7 8 8 11 12 12 19/30 Read block 255 len 28672 20/30 Read block 5 len 32768 21/30 Write block 153 len 24576 data: 10 12 9 12 14 13 22/30 Read block 219 len 40960 23/30 Flush 24/30 Write block 180 len 36864 data: 10 11 11 8 9 9 8 10 10 25/30 Read block 587 len 4096 26/30 Write block 203 len 16384 data: 12 13 12 13 27/30 Write block 378 len 36864 data: 6 5 4 5 6 6 5 6 7 28/30 Write block 489 len 28672 data: 11 11 8 10 11 12 11 29/30 Read block 477 len 12288 30/30 Read block 567 len 32768 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 8 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:15.938376876Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:15.938556284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1579,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 108 108 108 179 179 179 F F F 3 060-079 108 108 108 179 179 179 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 110 110 108 183 183 179 F F F <--- 8 160-179 110 110 107 183 183 177 F F F <--- 9 180-199 105 105 105 174 174 174 T T F <--- 10 200-219 110 110 105 183 183 174 T T F <--- 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 108 108 108 179 179 179 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 106 106 106 176 176 176 F F F 17 340-359 109 109 109 180 180 180 F F F 18 360-379 110 110 104 182 182 173 T T F <--- 19 380-399 110 110 95 183 183 157 T T F <--- 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 110 110 107 183 183 177 F F F <--- 23 460-479 107 107 107 177 177 177 F F F 24 480-499 102 102 102 168 168 168 T T F <--- 25 500-519 110 110 109 183 183 180 F F F <--- 26 520-539 109 109 109 180 180 180 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 108 108 108 178 178 178 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 110, Max flush: 183 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 108 108 108 179 179 179 F F F 3 060-079 108 108 108 179 179 179 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 110 110 110 183 183 183 F F F 8 160-179 110 110 110 183 183 183 F F F 9 180-199 111 111 111 184 184 184 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 108 108 108 179 179 179 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 106 106 106 176 176 176 F F F 17 340-359 109 109 109 180 180 180 F F F 18 360-379 111 111 111 184 184 184 F F F 19 380-399 111 111 111 184 184 184 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 110 110 110 183 183 183 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 111 111 111 184 184 184 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 109 109 109 180 180 180 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 108 108 108 178 178 178 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 111, Max flush: 184 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.187321831Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.195566428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.195908766Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.195943154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.195952473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.195964422Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Crucible b8f892d7-96bd-41fb-8ff0-3968bcbd1474 has session id: 94f4c6f5-00a3-4208-b70e-35b58c8520b5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196238624Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Upstairs opts: Upstairs UUID: b8f892d7-96bd-41fb-8ff0-3968bcbd1474, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196267152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Crucible stats registered with UUID: b8f892d7-96bd-41fb-8ff0-3968bcbd1474","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196278232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:20.196287501Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196410973Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196430442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196440821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.19660521Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196620329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"b8f892d7-96bd-41fb-8ff0-3968bcbd1474 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196652567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196664016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"0","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196673756Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"1","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196683485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"2","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196693124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196703094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196713093Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53821)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.19675471Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60346)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.19676572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61418)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.196776129Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197416157Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"1","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197446935Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"0","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197546928Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"2","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197761194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197808521Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[0]R flush_numbers[0..12]: [180, 177, 179, 179, 166, 177, 179, 183, 183, 184, 184, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197851408Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[0]R generation[0..12]: [109, 107, 108, 108, 100, 107, 108, 110, 110, 111, 111, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197864817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197876046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[1]R flush_numbers[0..12]: [180, 177, 179, 179, 166, 177, 179, 183, 183, 184, 184, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197886776Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[1]R generation[0..12]: [109, 107, 108, 108, 100, 107, 108, 110, 110, 111, 111, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197897445Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197908384Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[2]R flush_numbers[0..12]: [180, 177, 179, 179, 166, 177, 179, 183, 183, 184, 184, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197919064Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[2]R generation[0..12]: [109, 107, 108, 108, 100, 107, 108, 110, 110, 111, 111, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197929573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197940682Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197950002Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197959941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.19796941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.19797882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"":"downstairs","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197988529Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.197997468Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"0","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198008678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"1","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198018587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"client":"2","":"downstairs","session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198028846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198038076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"b8f892d7-96bd-41fb-8ff0-3968bcbd1474 is now active with session: 94f4c6f5-00a3-4208-b70e-35b58c8520b5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198048055Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198056775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"session_id":"94f4c6f5-00a3-4208-b70e-35b58c8520b5"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198065894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Activated sub_volume b8f892d7-96bd-41fb-8ff0-3968bcbd1474","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198073933Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198082043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.198374084Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"task":"crutest"} 01/30 Write block 520 len 20480 data: 10 6 5 7 6 02/30 Read block 218 len 36864 03/30 Read block 304 len 4096 04/30 Flush 05/30 Write block 490 len 32768 data: 12 9 11 12 13 12 11 12 06/30 Read block 387 len 24576 07/30 Write block 197 len 8192 data: 12 13 08/30 Flush 09/30 Write block 329 len 16384 data: 11 12 10 11 10/30 Write block 149 len 40960 data: 17 15 16 13 11 13 10 13 15 14 11/30 Write block 329 len 28672 data: 12 13 11 12 11 7 8 12/30 Read block 300 len 4096 13/30 Read block 451 len 8192 14/30 Read block 175 len 4096 15/30 Read block 303 len 36864 16/30 Read block 461 len 4096 17/30 Read block 162 len 24576 18/30 Read block 538 len 20480 19/30 Write block 366 len 12288 data: 15 14 14 20/30 Read block 559 len 16384 21/30 Write block 368 len 36864 data: 15 12 11 9 9 7 7 8 6 22/30 Write block 322 len 28672 data: 11 11 10 7 9 9 10 23/30 Read block 437 len 16384 24/30 Write block 150 len 20480 data: 16 17 14 12 14 25/30 Read block 214 len 40960 26/30 Read block 43 len 28672 27/30 Write block 274 len 16384 data: 10 7 7 4 28/30 Read block 382 len 12288 29/30 Write block 569 len 28672 data: 8 8 8 6 4 6 7 30/30 Write block 59 len 32768 data: 6 6 8 7 10 8 8 6 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 8 Done Done Done false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 25 29 Done 24 24 1 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:20.206876654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:20.207042094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1587,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 108 108 108 179 179 179 T T F <--- 3 060-079 108 108 108 179 179 179 T T F <--- 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 110 110 110 183 183 183 T T F <--- 8 160-179 110 110 110 183 183 183 F F F 9 180-199 112 112 111 186 186 184 F F F <--- 10 200-219 111 111 111 184 184 184 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 108 108 108 179 179 179 T T F <--- 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 106 106 106 176 176 176 T T F <--- 17 340-359 109 109 109 180 180 180 F F F 18 360-379 111 111 111 184 184 184 T T F <--- 19 380-399 111 111 111 184 184 184 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 110 110 110 183 183 183 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 112 112 111 186 186 184 F F T <--- 25 500-519 110 110 110 183 183 183 F F F 26 520-539 112 112 112 185 185 185 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 108 108 108 178 178 178 T T F <--- 29 580-599 104 104 104 173 173 173 F F F Max gen: 112, Max flush: 186 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 113 113 113 187 187 187 F F F 3 060-079 113 113 113 187 187 187 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 113 113 113 187 187 187 F F F 8 160-179 110 110 110 183 183 183 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 103 103 103 171 171 171 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 109 109 109 180 180 180 F F F 18 360-379 113 113 113 187 187 187 F F F 19 380-399 111 111 111 184 184 184 F F F 20 400-419 102 102 102 170 170 170 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 110 110 110 183 183 183 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 112 112 112 186 186 186 F F F 25 500-519 110 110 110 183 183 183 F F F 26 520-539 112 112 112 185 185 185 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 113 113 113 187 187 187 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 113, Max flush: 187 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.458188903Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.466376065Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.466659216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.466685124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.466693744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.466705423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Crucible 7720a92a-c6ee-49b5-850a-c11cf50225f1 has session id: 9c7ce01c-214b-4891-a836-5f36fc1de3c7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467137875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Upstairs opts: Upstairs UUID: 7720a92a-c6ee-49b5-850a-c11cf50225f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467161303Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Crucible stats registered with UUID: 7720a92a-c6ee-49b5-850a-c11cf50225f1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467171172Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:24.467180092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467337941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.46735339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.46736362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467472713Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467486812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"7720a92a-c6ee-49b5-850a-c11cf50225f1 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467519809Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467529359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"0","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467538988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"1","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467548818Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"2","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467558447Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467568446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467578296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45556)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.4676641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55068)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467690388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49115)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.467701887Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468339846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"2","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468378453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"0","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468398362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"1","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468624077Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468656135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[0]R flush_numbers[0..12]: [180, 177, 187, 187, 166, 177, 179, 187, 183, 186, 184, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468708551Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[0]R generation[0..12]: [109, 107, 113, 113, 100, 107, 108, 113, 110, 112, 111, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468719911Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.46873095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[1]R flush_numbers[0..12]: [180, 177, 187, 187, 166, 177, 179, 187, 183, 186, 184, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468741829Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[1]R generation[0..12]: [109, 107, 113, 113, 100, 107, 108, 113, 110, 112, 111, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468752638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468763648Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[2]R flush_numbers[0..12]: [180, 177, 187, 187, 166, 177, 179, 187, 183, 186, 184, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468776637Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[2]R generation[0..12]: [109, 107, 113, 113, 100, 107, 108, 113, 110, 112, 111, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468787276Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468798115Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468807455Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468817084Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468826514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468835823Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"":"downstairs","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468845602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468854332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"0","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468864031Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"1","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.4688736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"client":"2","":"downstairs","session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.46888359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468892749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"7720a92a-c6ee-49b5-850a-c11cf50225f1 is now active with session: 9c7ce01c-214b-4891-a836-5f36fc1de3c7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468902649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468911398Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pidDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":1595,"session_id":"9c7ce01c-214b-4891-a836-5f36fc1de3c7"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468928337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Activated sub_volume 7720a92a-c6ee-49b5-850a-c11cf50225f1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468936426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.468944626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.46918503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"task":"crutest"} 01/30 Flush 02/30 Write block 569 len 4096 data: 9 03/30 Write block 415 len 32768 data: 10 7 7 8 8 8 9 12 04/30 Read block 268 len 16384 05/30 Read block 69 len 20480 06/30 Write block 350 len 36864 data: 10 9 10 9 9 8 8 10 8 07/30 Read block 186 len 40960 08/30 Write block 516 len 8192 data: 13 12 09/30 Write block 52 len 12288 data: 8 9 8 10/30 Write block 426 len 12288 data: 12 11 8 11/30 Read block 69 len 32768 12/30 Read block 264 len 16384 13/30 Read block 575 len 16384 14/30 Write block 393 len 8192 data: 12 11 15/30 Write block 438 len 4096 data: 10 16/30 Read block 586 len 12288 17/30 Write block 378 len 12288 data: 7 6 5 18/30 Read block 552 len 28672 19/30 Read block 314 len 16384 20/30 Read block 379 len 20480 21/30 Read block 454 len 12288 22/30 Write block 444 len 12288 data: 8 9 9 23/30 Read block 344 len 16384 24/30 Write block 494 len 20480 data: 14 13 12 13 11 25/30 Flush 26/30 Write block 561 len 40960 data: 12 11 9 8 7 9 10 8 10 9 27/30 Read block 321 len 32768 28/30 Read block 333 len 12288 29/30 Write block 231 len 4096 data: 6 30/30 Read block 550 len 12288 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:24.47755238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:24.477740917Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1595,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 114 113 114 189 187 189 F F F <--- 3 060-079 113 113 113 187 187 187 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 113 113 113 187 187 187 F F F 8 160-179 110 110 110 183 183 183 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 103 103 103 171 171 171 T F T <--- 12 240-259 97 97 97 162 162 162 F F F 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 114 109 114 189 180 189 F F F <--- 18 360-379 114 113 114 189 187 189 F F F <--- 19 380-399 114 111 114 189 184 189 F F F <--- 20 400-419 114 102 114 189 170 189 F F F <--- 21 420-439 114 100 114 189 166 189 F F F <--- 22 440-459 114 110 114 189 183 189 F F F <--- 23 460-479 107 107 107 177 177 177 F F F 24 480-499 114 112 114 189 186 189 F F F <--- 25 500-519 114 110 114 189 183 189 F F F <--- 26 520-539 112 112 112 185 185 185 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 114 113 114 189 187 189 T F T <--- 29 580-599 104 104 104 173 173 173 F F F Max gen: 114, Max flush: 189 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 180 180 180 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 113 113 113 187 187 187 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 113 113 113 187 187 187 F F F 8 160-179 110 110 110 183 183 183 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 97 97 97 162 162 162 F F F 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 114 114 114 189 189 189 F F F 18 360-379 114 114 114 189 189 189 F F F 19 380-399 114 114 114 189 189 189 F F F 20 400-419 114 114 114 189 189 189 F F F 21 420-439 114 114 114 189 189 189 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 114 114 114 189 189 189 F F F 25 500-519 114 114 114 189 189 189 F F F 26 520-539 112 112 112 185 185 185 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 115 115 115 190 190 190 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 115, Max flush: 190 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.726529513Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.73478729Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735070001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735096649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735105369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735117228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Crucible ce0e3736-f7bc-48bd-b458-6c01ffe333e9 has session id: 0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735456546Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Upstairs opts: Upstairs UUID: ce0e3736-f7bc-48bd-b458-6c01ffe333e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735486404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Crucible stats registered with UUID: ce0e3736-f7bc-48bd-b458-6c01ffe333e9","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735497713Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:28.735507152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735652263Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735685201Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.73569648Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735809372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735825401Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"ce0e3736-f7bc-48bd-b458-6c01ffe333e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735859679Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735869459Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"0","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735879148Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"1","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735888937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"2","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735919915Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735942924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735953703Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42486)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735964412Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35614)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.735974992Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51297)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.736051767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.736645507Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"0","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.736693624Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"2","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.736719673Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"1","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.736957947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.736993565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[0]R flush_numbers[0..12]: [180, 177, 189, 187, 166, 177, 179, 187, 183, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737034512Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[0]R generation[0..12]: [109, 107, 114, 113, 100, 107, 108, 113, 110, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737046001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.73705721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[1]R flush_numbers[0..12]: [180, 177, 189, 187, 166, 177, 179, 187, 183, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.7370683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[1]R generation[0..12]: [109, 107, 114, 113, 100, 107, 108, 113, 110, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737079269Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737090588Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[2]R flush_numbers[0..12]: [180, 177, 189, 187, 166, 177, 179, 187, 183, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737101458Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[2]R generation[0..12]: [109, 107, 114, 113, 100, 107, 108, 113, 110, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737112267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737123276Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737132675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737142545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737154194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737163663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"":"downstairs","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737173733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737182622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"0","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737192752Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"1","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737202761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"client":"2","":"downstairs","session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.73721297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.73722228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"ce0e3736-f7bc-48bd-b458-6c01ffe333e9 is now active with session: 0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737232319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737241068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"session_id":"0cf98ad1-b0a4-44b4-8f10-50db40e3ec3c"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737250318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Activated sub_volume ce0e3736-f7bc-48bd-b458-6c01ffe333e9","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737258387Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737266587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.737622133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"task":"crutest"} 01/30 Flush 02/30 Write block 520 len 4096 data: 11 03/30 Read block 459 len 24576 04/30 Read block 545 len 20480 05/30 Read block 134 len 4096 06/30 Write block 492 len 16384 data: 12 13 15 14 07/30 Write block 497 len 28672 data: 14 12 11 7 8 6 8 08/30 Write block 374 len 32768 data: 8 9 7 6 8 7 6 6 09/30 Read block 416 len 4096 10/30 Write block 241 len 20480 data: 7 8 7 7 8 11/30 Read block 522 len 28672 12/30 Read block 213 len 40960 13/30 Read block 183 len 28672 14/30 Read block 467 len 40960 15/30 Read block 195 len 24576 16/30 Read block 495 len 32768 17/30 Read block 302 len 16384 18/30 Write block 14 len 4096 data: 7 19/30 Read block 451 len 24576 20/30 Read block 473 len 40960 21/30 Write block 155 len 24576 data: 11 14 16 15 15 12 22/30 Read block 102 len 12288 23/30 Read block 51 len 24576 24/30 Read block 185 len 8192 25/30 Read block 101 len 24576 26/30 Flush 27/30 Write block 343 len 28672 data: 11 11 13 14 14 12 10 28/30 Read block 48 len 12288 29/30 Read block 526 len 4096 30/30 Write block 338 len 12288 data: 10 9 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:28.746363548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:28.746541237Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1603,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 109 192 192 180 F F F <--- 1 020-039 107 107 107 177 177 177 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 113 113 113 187 187 187 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 116 116 113 192 192 187 F F F <--- 8 160-179 116 116 110 192 192 183 F F F <--- 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 116 116 97 192 192 162 F F F <--- 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 113 113 113 187 187 187 T T F <--- 17 340-359 114 114 114 189 189 189 T T F <--- 18 360-379 116 116 114 192 192 189 F F F <--- 19 380-399 116 116 114 192 192 189 F F F <--- 20 400-419 114 114 114 189 189 189 F F F 21 420-439 114 114 114 189 189 189 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 116 116 114 192 192 189 F F F <--- 25 500-519 116 116 114 192 192 189 F F F <--- 26 520-539 116 116 112 192 192 185 F F F <--- 27 540-559 108 108 108 178 178 178 F F F 28 560-579 115 115 115 190 190 190 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 116, Max flush: 192 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 192 192 192 F F F 1 020-039 107 107 107 177 177 177 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 113 113 113 187 187 187 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 116 116 116 192 192 192 F F F 8 160-179 116 116 116 192 192 192 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 116 116 116 192 192 192 F F F 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 107 107 177 177 177 F F F 16 320-339 117 117 117 193 193 193 F F F 17 340-359 117 117 117 193 193 193 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 116 116 116 192 192 192 F F F 20 400-419 114 114 114 189 189 189 F F F 21 420-439 114 114 114 189 189 189 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 116 116 116 192 192 192 F F F 25 500-519 116 116 116 192 192 192 F F F 26 520-539 116 116 116 192 192 192 F F F 27 540-559 108 108 108 178 178 178 F F F 28 560-579 115 115 115 190 190 190 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 117, Max flush: 193 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:32.996353584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:33.003653334Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.00370704Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.010768216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011024789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011050367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011058917Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011070516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Crucible ac78ab2f-81ee-43aa-8aa1-59abd98bb548 has session id: f0d37512-efda-4b3b-92f6-443f24be553f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011436792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Upstairs opts: Upstairs UUID: ac78ab2f-81ee-43aa-8aa1-59abd98bb548, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011458401Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Crucible stats registered with UUID: ac78ab2f-81ee-43aa-8aa1-59abd98bb548","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.01146877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:33.011478009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.01162415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011639089Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011649228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011796238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011812737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"ac78ab2f-81ee-43aa-8aa1-59abd98bb548 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011836406Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011847915Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"0","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011857584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"1","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011867284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"2","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011876923Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011886852Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011896842Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62270)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011952538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39132)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011963467Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59474)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.011981266Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.012701339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"1","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.012720738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"2","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.012731967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"0","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013060335Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013102122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[0]R flush_numbers[0..12]: [192, 177, 189, 187, 166, 177, 179, 192, 192, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013167238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[0]R generation[0..12]: [116, 107, 114, 113, 100, 107, 108, 116, 116, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013181067Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013192177Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[1]R flush_numbers[0..12]: [192, 177, 189, 187, 166, 177, 179, 192, 192, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013203006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[1]R generation[0..12]: [116, 107, 114, 113, 100, 107, 108, 116, 116, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013213705Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013224714Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[2]R flush_numbers[0..12]: [192, 177, 189, 187, 166, 177, 179, 192, 192, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013235404Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[2]R generation[0..12]: [116, 107, 114, 113, 100, 107, 108, 116, 116, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013246133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013257122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013266812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013276691Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.01328615Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.01329586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"":"downstairs","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013305619Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013314428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"0","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013325878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"1","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013335727Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"client":"2","":"downstairs","session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013345796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013355056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"ac78ab2f-81ee-43aa-8aa1-59abd98bb548 is now active with session: f0d37512-efda-4b3b-92f6-443f24be553f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013365025Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013373875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"session_id":"f0d37512-efda-4b3b-92f6-443f24be553f"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013382894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Activated sub_volume ac78ab2f-81ee-43aa-8aa1-59abd98bb548","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013391023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013399153Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.013697333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} 01/30 Write block 13 len 40960 data: 6 8 6 6 8 9 10 10 11 11 02/30 Write block 71 len 20480 data: 12 13 12 12 13 03/30 Write block 313 len 36864 data: 8 8 6 8 7 8 11 10 10 04/30 Flush 05/30 Write block 405 len 8192 data: 10 8 06/30 Write block 537 len 24576 data: 10 11 10 11 11 11 07/30 Read block 363 len 36864 08/30 Flush 09/30 Write block 3 len 32768 data: 7 8 10 12 12 12 8 10 10/30 Read block 183 len 32768 11/30 Write block 563 len 28672 data: 10 9 8 10 11 9 11 12/30 Read block 0 len 4096 13/30 Write block 448 len 32768 data: 13 13 11 10 9 10 9 8 14/30 Write block 435 len 24576 data: 9 9 9 11 8 7 15/30 Read block 334 len 32768 16/30 Read block 89 len 4096 17/30 Read block 415 len 8192 18/30 Flush 19/30 Write block 70 len 32768 data: 12 13 14 13 13 14 13 10 20/30 Read block 290 len 12288 21/30 Flush 22/30 Write block 564 len 20480 data: 10 9 11 12 10 23/30 Read block 340 len 40960 24/30 Read block 533 len 8192 25/30 Read block 595 len 16384 26/30 Read block 32 len 4096 27/30 Write block 242 len 36864 data: 9 8 8 9 9 10 9 9 12 28/30 Write block 383 len 8192 data: 7 6 29/30 Read block 48 len 24576 30/30 Read block 19 len 8192 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:33.022783806Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:33.022946125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1611,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 118 118 192 196 196 T F F <--- 1 020-039 107 118 118 177 194 194 T F F <--- 2 040-059 114 114 114 189 189 189 F F F 3 060-079 113 118 118 187 197 197 F F F <--- 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 116 116 116 192 192 192 F F F 8 160-179 116 116 116 192 192 192 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 116 116 116 192 192 192 F T T <--- 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 107 118 118 177 194 194 F F F <--- 16 320-339 117 118 118 193 194 194 F F F <--- 17 340-359 117 117 117 193 193 193 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 116 116 116 192 192 192 F T T <--- 20 400-419 114 118 118 189 195 195 F F F <--- 21 420-439 114 118 118 189 196 196 F F F <--- 22 440-459 114 118 118 189 196 196 F F F <--- 23 460-479 107 107 107 177 177 177 F F F 24 480-499 116 116 116 192 192 192 F F F 25 500-519 116 116 116 192 192 192 F F F 26 520-539 116 118 118 192 195 195 F F F <--- 27 540-559 108 118 118 178 195 195 F F F <--- 28 560-579 115 118 118 190 196 196 F T T <--- 29 580-599 104 104 104 173 173 173 F F F Max gen: 118, Max flush: 197 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 118 118 118 194 194 194 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 118 118 118 197 197 197 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 116 116 116 192 192 192 F F F 8 160-179 116 116 116 192 192 192 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 119 119 119 198 198 198 F F F 13 260-279 113 113 113 187 187 187 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 118 118 118 194 194 194 F F F 16 320-339 118 118 118 194 194 194 F F F 17 340-359 117 117 117 193 193 193 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 119 119 119 198 198 198 F F F 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 118 118 118 196 196 196 F F F 23 460-479 107 107 107 177 177 177 F F F 24 480-499 116 116 116 192 192 192 F F F 25 500-519 116 116 116 192 192 192 F F F 26 520-539 118 118 118 195 195 195 F F F 27 540-559 118 118 118 195 195 195 F F F 28 560-579 119 119 119 198 198 198 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 119, Max flush: 198 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.284791871Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:37.292042474Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.29209621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299025425Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299282358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299308426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299319635Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299331555Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Crucible 7f368dee-8d7b-4c9d-b756-82938d05f56c has session id: 6a6cc960-e2cf-49c8-a75a-2f083455b4c0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299748827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Upstairs opts: Upstairs UUID: 7f368dee-8d7b-4c9d-b756-82938d05f56c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299778455Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Crucible stats registered with UUID: 7f368dee-8d7b-4c9d-b756-82938d05f56c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299789684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:37.299799154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299918446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299936225Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.299946264Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300142651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.30016618Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"7f368dee-8d7b-4c9d-b756-82938d05f56c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300187728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300197028Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"0","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300207267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"1","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300217146Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"2","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300233075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300243375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300253324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:32894)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300321539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39403)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300393885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38008)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300405444Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.300955848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"1","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301003095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"2","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.3010706Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"0","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301286826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301307405Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[0]R flush_numbers[0..12]: [196, 194, 189, 197, 166, 177, 179, 192, 192, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301326193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[0]R generation[0..12]: [118, 118, 114, 118, 100, 107, 108, 116, 116, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301337453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301348602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[1]R flush_numbers[0..12]: [196, 194, 189, 197, 166, 177, 179, 192, 192, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301359431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[1]R generation[0..12]: [118, 118, 114, 118, 100, 107, 108, 116, 116, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.3013732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.30138463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[2]R flush_numbers[0..12]: [196, 194, 189, 197, 166, 177, 179, 192, 192, 186, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301395569Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[2]R generation[0..12]: [118, 118, 114, 118, 100, 107, 108, 116, 116, 112, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301406328Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301417357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301426787Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301436556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301446115Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301455655Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"":"downstairs","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301470484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301479283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"0","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301488943Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"1","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301498462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"client":"2","":"downstairs","session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301508261Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301517391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"7f368dee-8d7b-4c9d-b756-82938d05f56c is now active with session: 6a6cc960-e2cf-49c8-a75a-2f083455b4c0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.30152883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301537719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"session_id":"6a6cc960-e2cf-49c8-a75a-2f083455b4c0"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301546839Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Activated sub_volume 7f368dee-8d7b-4c9d-b756-82938d05f56c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301562878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.301571127Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.30184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} 01/30 Flush 02/30 Write block 186 len 12288 data: 9 11 11 03/30 Flush 04/30 Write block 445 len 20480 data: 10 10 12 14 14 05/30 Read block 21 len 12288 06/30 Read block 567 len 36864 07/30 Read block 2 len 36864 08/30 Flush 09/30 Write block 441 len 12288 data: 8 9 8 10/30 Flush 11/30 Write block 259 len 28672 data: 4 5 8 6 5 7 7 12/30 Read block 439 len 28672 13/30 Write block 38 len 40960 data: 8 6 6 6 8 8 8 7 7 8 14/30 Read block 6 len 32768 15/30 Read block 94 len 32768 16/30 Write block 145 len 20480 data: 10 11 14 14 18 17/30 Read block 533 len 28672 18/30 Write block 188 len 12288 data: 12 9 9 19/30 Read block 361 len 16384 20/30 Read block 583 len 32768 21/30 Read block 121 len 4096 22/30 Write block 347 len 36864 data: 15 13 11 11 10 11 10 10 9 23/30 Flush 24/30 Write block 161 len 16384 data: 12 10 8 6 25/30 Write block 265 len 28672 data: 8 9 7 10 9 9 8 26/30 Read block 499 len 28672 27/30 Write block 454 len 40960 data: 10 9 9 7 7 9 9 6 7 7 28/30 Write block 311 len 32768 data: 11 10 9 9 7 9 8 9 29/30 Write block 23 len 40960 data: 10 8 7 7 6 7 7 6 7 7 30/30 Write block 563 len 12288 data: 11 11 10 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 10 Sent Done Sent false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 3 4 32 Done 2 24 23 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:37.311643395Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:37.311832142Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1619,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 118 120 120 194 203 203 F T T <--- 2 040-059 114 120 120 189 203 203 F F F <--- 3 060-079 118 118 118 197 197 197 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 116 120 120 192 203 203 F F F <--- 8 160-179 116 116 116 192 192 192 F T T <--- 9 180-199 120 120 120 200 203 203 F F F <--- 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 119 120 120 198 203 203 F F F <--- 13 260-279 113 120 120 187 203 203 F T T <--- 14 280-299 107 107 107 177 177 177 F F F 15 300-319 118 118 118 194 194 194 F T T <--- 16 320-339 118 118 118 194 194 194 F F F 17 340-359 117 120 120 193 203 203 F F F <--- 18 360-379 116 116 116 192 192 192 F F F 19 380-399 119 119 119 198 198 198 F F F 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 118 120 120 196 202 202 T T T <--- 23 460-479 107 107 107 177 177 177 F T T <--- 24 480-499 116 116 116 192 192 192 F F F 25 500-519 116 116 116 192 192 192 F F F 26 520-539 118 118 118 195 195 195 F F F 27 540-559 118 118 118 195 195 195 F F F 28 560-579 119 119 119 198 198 198 F T T <--- 29 580-599 104 104 104 173 173 173 F F F Max gen: 120, Max flush: 203 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 196 196 196 F F F 1 020-039 121 121 121 204 204 204 F F F 2 040-059 120 120 120 203 203 203 F F F 3 060-079 118 118 118 197 197 197 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 107 107 107 177 177 177 F F F 6 120-139 108 108 108 179 179 179 F F F 7 140-159 120 120 120 203 203 203 F F F 8 160-179 121 121 121 204 204 204 F F F 9 180-199 120 120 120 203 203 203 F F F 10 200-219 111 111 111 184 184 184 F F F 11 220-239 115 115 115 190 190 190 F F F 12 240-259 120 120 120 203 203 203 F F F 13 260-279 121 121 121 204 204 204 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 121 121 121 204 204 204 F F F 16 320-339 118 118 118 194 194 194 F F F 17 340-359 120 120 120 203 203 203 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 119 119 119 198 198 198 F F F 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 121 121 121 204 204 204 F F F 23 460-479 121 121 121 204 204 204 F F F 24 480-499 116 116 116 192 192 192 F F F 25 500-519 116 116 116 192 192 192 F F F 26 520-539 118 118 118 195 195 195 F F F 27 540-559 118 118 118 195 195 195 F F F 28 560-579 121 121 121 204 204 204 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 121, Max flush: 204 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.573675399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582104545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582388786Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582415625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582424254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582436103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Crucible 4a5b352c-7ff3-41ce-8778-808c788b6095 has session id: 26b4cff7-cc13-4194-8cad-5d020353afda","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582798409Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Upstairs opts: Upstairs UUID: 4a5b352c-7ff3-41ce-8778-808c788b6095, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582828237Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Crucible stats registered with UUID: 4a5b352c-7ff3-41ce-8778-808c788b6095","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582839307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:41.582851126Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.582982537Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583022515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583033584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583152126Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583170765Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"4a5b352c-7ff3-41ce-8778-808c788b6095 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583200123Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583209622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"0","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583219432Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"1","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583229001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"2","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.58323839Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.58324869Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583258489Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"ds_connection connected from Ok(127.0.0.1:61721)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583297167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"ds_connection connected from Ok(127.0.0.1:46752)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583310686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"ds_connection connected from Ok(127.0.0.1:53541)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.58338964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583943934Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"0","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.583985841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"2","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584049317Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"1","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584263583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584281162Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[0]R flush_numbers[0..12]: [196, 204, 203, 197, 166, 177, 179, 203, 204, 203, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584334438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[0]R generation[0..12]: [118, 121, 120, 118, 100, 107, 108, 120, 121, 120, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584345768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584356907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[1]R flush_numbers[0..12]: [196, 204, 203, 197, 166, 177, 179, 203, 204, 203, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584367656Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[1]R generation[0..12]: [118, 121, 120, 118, 100, 107, 108, 120, 121, 120, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584378295Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584389315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[2]R flush_numbers[0..12]: [196, 204, 203, 197, 166, 177, 179, 203, 204, 203, 184, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584400214Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[2]R generation[0..12]: [118, 121, 120, 118, 100, 107, 108, 120, 121, 120, 111, 115]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584410843Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584421953Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584434742Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584444561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.58445397Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.58446368Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"":"downstairs","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584473519Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584482379Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"0","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584492158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"1","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584501767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"client":"2","":"downstairs","session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584511717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584520866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"4a5b352c-7ff3-41ce-8778-808c788b6095 is now active with session: 26b4cff7-cc13-4194-8cad-5d020353afda","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584530885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584539665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"session_id":"26b4cff7-cc13-4194-8cad-5d020353afda"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584548724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Activated sub_volume 4a5b352c-7ff3-41ce-8778-808c788b6095","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584556744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584564823Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.584849954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"task":"crutest"} 01/30 Write block 536 len 24576 data: 10 11 12 11 12 12 02/30 Read block 347 len 36864 03/30 Flush 04/30 Write block 477 len 32768 data: 10 10 11 10 11 12 10 10 05/30 Flush 06/30 Flush 07/30 Write block 158 len 16384 data: 16 16 13 13 08/30 Read block 440 len 24576 09/30 Read block 394 len 12288 10/30 Read block 344 len 28672 11/30 Write block 110 len 8192 data: 9 10 12/30 Read block 505 len 28672 13/30 Write block 120 len 28672 data: 12 10 11 12 12 11 11 14/30 Read block 318 len 40960 15/30 Read block 519 len 28672 16/30 Read block 391 len 12288 17/30 Write block 259 len 24576 data: 5 6 9 7 6 8 18/30 Flush 19/30 Write block 13 len 8192 data: 7 9 20/30 Read block 371 len 16384 21/30 Write block 576 len 12288 data: 9 9 7 22/30 Write block 193 len 36864 data: 7 8 7 9 13 14 13 11 9 23/30 Write block 239 len 40960 data: 10 9 8 10 9 9 10 10 11 10 24/30 Write block 326 len 32768 data: 10 10 11 13 14 12 13 12 25/30 Flush 26/30 Write block 372 len 32768 data: 10 8 9 10 8 7 9 8 27/30 Write block 469 len 8192 data: 7 8 28/30 Write block 519 len 40960 data: 10 12 7 6 8 7 7 7 7 6 29/30 Read block 84 len 20480 30/30 Read block 209 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 8 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 27 0 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1002)) Some(JobId(1024)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:41.594376718Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:41.594556006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1627,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 118 122 209 196 209 F F F <--- 1 020-039 121 121 121 204 204 204 F F F 2 040-059 120 120 120 203 203 203 F F F 3 060-079 118 118 118 197 197 197 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 122 107 122 208 177 208 F F F <--- 6 120-139 122 108 122 208 179 208 F F F <--- 7 140-159 122 120 122 208 203 208 F F F <--- 8 160-179 122 121 122 208 204 208 F F F <--- 9 180-199 122 120 122 209 203 209 F F F <--- 10 200-219 122 111 122 209 184 209 F F F <--- 11 220-239 122 115 122 209 190 209 F F F <--- 12 240-259 122 120 122 209 203 209 F F F <--- 13 260-279 122 121 122 208 204 208 F F F <--- 14 280-299 107 107 107 177 177 177 F F F 15 300-319 121 121 121 204 204 204 F F F 16 320-339 122 118 122 209 194 209 F F F <--- 17 340-359 120 120 120 203 203 203 F F F 18 360-379 116 116 116 192 192 192 T F T <--- 19 380-399 119 119 119 198 198 198 F F F 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 121 121 121 204 204 204 F F F 23 460-479 122 121 122 206 204 206 T F T <--- 24 480-499 122 116 122 206 192 206 F F F <--- 25 500-519 116 116 116 192 192 192 T F T <--- 26 520-539 122 122 122 205 205 205 T F T <--- 27 540-559 122 122 122 205 205 205 F F F 28 560-579 122 121 122 209 204 209 F F F <--- 29 580-599 104 104 104 173 173 173 F F F Max gen: 122, Max flush: 209 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 121 121 121 204 204 204 F F F 2 040-059 120 120 120 203 203 203 F F F 3 060-079 118 118 118 197 197 197 F F F 4 080-099 100 100 100 166 166 166 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 122 122 122 208 208 208 F F F 8 160-179 122 122 122 208 208 208 F F F 9 180-199 122 122 122 209 209 209 F F F 10 200-219 122 122 122 209 209 209 F F F 11 220-239 122 122 122 209 209 209 F F F 12 240-259 122 122 122 209 209 209 F F F 13 260-279 122 122 122 208 208 208 F F F 14 280-299 107 107 107 177 177 177 F F F 15 300-319 121 121 121 204 204 204 F F F 16 320-339 122 122 122 209 209 209 F F F 17 340-359 120 120 120 203 203 203 F F F 18 360-379 123 123 123 210 210 210 F F F 19 380-399 119 119 119 198 198 198 F F F 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 121 121 121 204 204 204 F F F 23 460-479 123 123 123 210 210 210 F F F 24 480-499 122 122 122 206 206 206 F F F 25 500-519 123 123 123 210 210 210 F F F 26 520-539 123 123 123 210 210 210 F F F 27 540-559 122 122 122 205 205 205 F F F 28 560-579 122 122 122 209 209 209 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 123, Max flush: 210 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.859361757Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.867670361Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.867938573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.867965651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.867974601Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.86798651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Crucible 4fde9d6b-ceb2-4deb-af34-2dfee584e6ff has session id: a4a97a8d-20ff-4638-b307-71c85b4497de","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868302539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Upstairs opts: Upstairs UUID: 4fde9d6b-ceb2-4deb-af34-2dfee584e6ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868361715Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Crucible stats registered with UUID: 4fde9d6b-ceb2-4deb-af34-2dfee584e6ff","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868387024Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:45.868397063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868515365Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868531054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868541124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868694143Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868718192Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"4fde9d6b-ceb2-4deb-af34-2dfee584e6ff active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868737731Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.86874676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"0","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868756789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"1","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868766659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"2","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868776308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868791257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868801086Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"ds_connection connected from Ok(127.0.0.1:55394)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868818015Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"ds_connection connected from Ok(127.0.0.1:49225)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.86889218Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"ds_connection connected from Ok(127.0.0.1:64708)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.868925778Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.86950643Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"1","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869530288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"0","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869598534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"2","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869842798Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869852417Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[0]R flush_numbers[0..12]: [209, 204, 203, 197, 166, 208, 208, 208, 208, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869879246Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[0]R generation[0..12]: [122, 121, 120, 118, 100, 122, 122, 122, 122, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869890645Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869901734Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[1]R flush_numbers[0..12]: [209, 204, 203, 197, 166, 208, 208, 208, 208, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869912373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[1]R generation[0..12]: [122, 121, 120, 118, 100, 122, 122, 122, 122, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869923133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869934102Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[2]R flush_numbers[0..12]: [209, 204, 203, 197, 166, 208, 208, 208, 208, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869944901Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[2]R generation[0..12]: [122, 121, 120, 118, 100, 122, 122, 122, 122, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.86995554Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.86996668Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869976319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869986218Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.869995678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870005247Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"":"downstairs","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870015257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870024076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"0","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870035625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"1","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870045435Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"client":"2","":"downstairs","session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870055414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870064553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"4fde9d6b-ceb2-4deb-af34-2dfee584e6ff is now active with session: a4a97a8d-20ff-4638-b307-71c85b4497de","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870074533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870083252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"session_id":"a4a97a8d-20ff-4638-b307-71c85b4497de"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870092391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Activated sub_volume 4fde9d6b-ceb2-4deb-af34-2dfee584e6ff","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.870100431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.87010856Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.87041274Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"task":"crutest"} 01/30 Flush 02/30 Write block 385 len 36864 data: 7 8 8 9 8 10 11 12 13 03/30 Write block 350 len 24576 data: 12 11 12 11 11 10 04/30 Write block 26 len 4096 data: 8 05/30 Write block 235 len 12288 data: 9 9 10 06/30 Read block 297 len 20480 07/30 Write block 528 len 40960 data: 7 8 9 9 9 8 7 9 11 12 08/30 Read block 367 len 16384 09/30 Read block 310 len 28672 10/30 Flush 11/30 Flush 12/30 Flush 13/30 Write block 498 len 40960 data: 13 12 8 9 7 9 9 9 8 6 14/30 Flush 15/30 Write block 86 len 28672 data: 5 6 6 7 8 9 8 16/30 Write block 144 len 28672 data: 10 11 12 15 15 19 17 17/30 Write block 268 len 28672 data: 11 10 10 9 9 10 11 18/30 Read block 4 len 28672 19/30 Read block 48 len 4096 20/30 Write block 289 len 32768 data: 10 9 9 6 5 7 7 6 21/30 Write block 86 len 20480 data: 6 7 7 8 9 22/30 Read block 457 len 28672 23/30 Read block 592 len 12288 24/30 Read block 139 len 24576 25/30 Write block 239 len 8192 data: 11 10 26/30 Write block 392 len 12288 data: 13 14 12 27/30 Write block 184 len 28672 data: 10 10 10 12 13 10 10 28/30 Read block 57 len 4096 29/30 Write block 171 len 40960 data: 9 6 8 9 8 8 10 9 10 11 30/30 Read block 137 len 20480 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 26 27 Done 28 29 3 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:45.879095659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:45.879290597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1635,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 124 124 121 212 212 204 F F T <--- 2 040-059 120 120 120 203 203 203 F F F 3 060-079 118 118 118 197 197 197 F F F 4 080-099 100 100 100 166 166 166 T T F <--- 5 100-119 122 122 122 208 208 208 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 122 122 122 208 208 208 T T F <--- 8 160-179 122 122 122 208 208 208 T T F <--- 9 180-199 122 122 122 209 209 209 T T F <--- 10 200-219 122 122 122 209 209 209 F F F 11 220-239 124 124 122 212 212 209 T T F <--- 12 240-259 122 122 122 209 209 209 T T F <--- 13 260-279 122 122 122 208 208 208 T T F <--- 14 280-299 107 107 107 177 177 177 T T F <--- 15 300-319 121 121 121 204 204 204 F F F 16 320-339 122 122 122 209 209 209 F F F 17 340-359 124 124 120 212 212 203 F F T <--- 18 360-379 123 123 123 210 210 210 F F F 19 380-399 124 124 119 212 212 198 T T T <--- 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 121 121 121 204 204 204 F F F 23 460-479 123 123 123 210 210 210 F F F 24 480-499 124 124 122 215 215 206 F F F <--- 25 500-519 124 124 123 215 215 210 F F F <--- 26 520-539 124 124 123 212 212 210 F F F <--- 27 540-559 122 122 122 205 205 205 F F F 28 560-579 122 122 122 209 209 209 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 124, Max flush: 215 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 124 124 124 212 212 212 F F F 2 040-059 120 120 120 203 203 203 F F F 3 060-079 118 118 118 197 197 197 F F F 4 080-099 125 125 125 216 216 216 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 125 125 125 216 216 216 F F F 8 160-179 125 125 125 216 216 216 F F F 9 180-199 125 125 125 216 216 216 F F F 10 200-219 122 122 122 209 209 209 F F F 11 220-239 125 125 125 216 216 216 F F F 12 240-259 125 125 125 216 216 216 F F F 13 260-279 125 125 125 216 216 216 F F F 14 280-299 125 125 125 216 216 216 F F F 15 300-319 121 121 121 204 204 204 F F F 16 320-339 122 122 122 209 209 209 F F F 17 340-359 124 124 124 212 212 212 F F F 18 360-379 123 123 123 210 210 210 F F F 19 380-399 125 125 125 216 216 216 F F F 20 400-419 118 118 118 195 195 195 F F F 21 420-439 118 118 118 196 196 196 F F F 22 440-459 121 121 121 204 204 204 F F F 23 460-479 123 123 123 210 210 210 F F F 24 480-499 124 124 124 215 215 215 F F F 25 500-519 124 124 124 215 215 215 F F F 26 520-539 124 124 124 212 212 212 F F F 27 540-559 122 122 122 205 205 205 F F F 28 560-579 122 122 122 209 209 209 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 125, Max flush: 216 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.140042035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.148307932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.148602512Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.148627881Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.14863647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.148651589Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"Crucible 41c887b6-e665-4120-942a-47d4b04b7388 has session id: 84e7019d-3e15-4bee-a511-cc7ac8af72e7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.148992007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Upstairs opts: Upstairs UUID: 41c887b6-e665-4120-942a-47d4b04b7388, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149033594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Crucible stats registered with UUID: 41c887b6-e665-4120-942a-47d4b04b7388","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149044063Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:50.149053163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149196733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149211642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149221622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149360452Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149372072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"41c887b6-e665-4120-942a-47d4b04b7388 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149388491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.1493974Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"0","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149407359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"1","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149416999Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"2","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149426628Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149438567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149448567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56874)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149488654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36090)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149533921Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:32877)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.149559709Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150188318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"1","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150214866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"2","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150231975Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"0","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150555814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150579402Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[0]R flush_numbers[0..12]: [209, 212, 203, 197, 216, 208, 208, 216, 216, 216, 209, 216]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150590202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[0]R generation[0..12]: [122, 124, 120, 118, 125, 122, 122, 125, 125, 125, 122, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150601561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.15061279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[1]R flush_numbers[0..12]: [209, 212, 203, 197, 216, 208, 208, 216, 216, 216, 209, 216]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150623809Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[1]R generation[0..12]: [122, 124, 120, 118, 125, 122, 122, 125, 125, 125, 122, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150634659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150649088Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[2]R flush_numbers[0..12]: [209, 212, 203, 197, 216, 208, 208, 216, 216, 216, 209, 216]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150659977Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[2]R generation[0..12]: [122, 124, 120, 118, 125, 122, 122, 125, 125, 125, 122, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150670696Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150681686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150691315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150701314Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150710604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150719963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"":"downstairs","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150729602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150738422Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"0","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150747961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"1","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150757541Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"client":"2","":"downstairs","session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.15076754Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150776749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"41c887b6-e665-4120-942a-47d4b04b7388 is now active with session: 84e7019d-3e15-4bee-a511-cc7ac8af72e7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150786599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150797438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"session_id":"84e7019d-3e15-4bee-a511-cc7ac8af72e7"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150806567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"Activated sub_volume 41c887b6-e665-4120-942a-47d4b04b7388","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150814647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.150827486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.151083519Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"task":"crutest"} 01/30 Write block 529 len 40960 data: 9 10 10 10 9 8 10 12 13 13 02/30 Write block 307 len 8192 data: 11 11 03/30 Write block 306 len 40960 data: 10 12 12 11 14 12 11 10 10 8 04/30 Write block 379 len 36864 data: 9 7 7 7 8 7 8 9 9 05/30 Read block 364 len 28672 06/30 Read block 526 len 36864 07/30 Read block 386 len 24576 08/30 Flush 09/30 Write block 82 len 40960 data: 5 5 6 4 7 8 8 9 10 10 10/30 Read block 594 len 4096 11/30 Write block 353 len 16384 data: 12 12 11 9 12/30 Write block 263 len 16384 data: 7 9 9 10 13/30 Read block 65 len 4096 14/30 Write block 318 len 24576 data: 10 12 11 11 12 12 15/30 Read block 549 len 36864 16/30 Read block 392 len 20480 17/30 Write block 68 len 4096 data: 7 18/30 Flush 19/30 Write block 57 len 24576 data: 6 4 7 7 9 8 20/30 Flush 21/30 Write block 334 len 24576 data: 8 9 10 9 11 10 22/30 Write block 88 len 36864 data: 9 10 11 11 9 7 7 8 8 23/30 Flush 24/30 Write block 79 len 16384 data: 6 6 6 6 25/30 Write block 410 len 20480 data: 9 10 10 11 10 26/30 Write block 286 len 36864 data: 10 9 12 11 10 10 7 6 8 27/30 Read block 556 len 20480 28/30 Write block 417 len 32768 data: 8 9 9 9 10 13 9 10 29/30 Write block 467 len 32768 data: 7 6 8 9 10 11 9 7 30/30 Read block 382 len 16384 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 2 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 29 30 Done 29 30 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:50.159857802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:50.16003757Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1643,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 124 124 124 212 212 212 F F F 2 040-059 126 126 120 219 219 203 F F F <--- 3 060-079 126 126 118 219 219 197 T T F <--- 4 080-099 126 126 125 220 220 216 T T F <--- 5 100-119 122 122 122 208 208 208 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 125 125 125 216 216 216 F F F 8 160-179 125 125 125 216 216 216 F F F 9 180-199 125 125 125 216 216 216 F F F 10 200-219 122 122 122 209 209 209 F F F 11 220-239 125 125 125 216 216 216 F F F 12 240-259 125 125 125 216 216 216 F F F 13 260-279 126 126 125 218 218 216 F F F <--- 14 280-299 125 125 125 216 216 216 T T F <--- 15 300-319 126 126 121 218 218 204 F F F <--- 16 320-339 126 126 122 220 220 209 F F F <--- 17 340-359 126 126 124 218 218 212 F F F <--- 18 360-379 126 126 123 217 217 210 F F F <--- 19 380-399 126 126 125 217 217 216 F F F <--- 20 400-419 118 118 118 195 195 195 T T F <--- 21 420-439 118 118 118 196 196 196 T T F <--- 22 440-459 121 121 121 204 204 204 F F F 23 460-479 123 123 123 210 210 210 T T F <--- 24 480-499 124 124 124 215 215 215 F F F 25 500-519 124 124 124 215 215 215 F F F 26 520-539 126 126 124 217 217 212 F F T <--- 27 540-559 122 122 122 205 205 205 F F F 28 560-579 122 122 122 209 209 209 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 126, Max flush: 220 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 124 124 124 212 212 212 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 127 127 127 221 221 221 F F F 4 080-099 127 127 127 221 221 221 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 125 125 125 216 216 216 F F F 8 160-179 125 125 125 216 216 216 F F F 9 180-199 125 125 125 216 216 216 F F F 10 200-219 122 122 122 209 209 209 F F F 11 220-239 125 125 125 216 216 216 F F F 12 240-259 125 125 125 216 216 216 F F F 13 260-279 126 126 126 218 218 218 F F F 14 280-299 127 127 127 221 221 221 F F F 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 126 126 126 217 217 217 F F F 19 380-399 126 126 126 217 217 217 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 127 127 127 221 221 221 F F F 22 440-459 121 121 121 204 204 204 F F F 23 460-479 127 127 127 221 221 221 F F F 24 480-499 124 124 124 215 215 215 F F F 25 500-519 124 124 124 215 215 215 F F F 26 520-539 126 126 126 217 217 217 F F F 27 540-559 122 122 122 205 205 205 F F F 28 560-579 122 122 122 209 209 209 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 127, Max flush: 221 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.418295622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:54.425537716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.425590202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.432605211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.432855764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.432884982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.432893602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.432905221Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Crucible 352c04d7-6cc7-4726-9d7e-a7fa99cfb313 has session id: d673197d-2513-4307-aa47-b528b161213d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433214241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Upstairs opts: Upstairs UUID: 352c04d7-6cc7-4726-9d7e-a7fa99cfb313, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433234099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Crucible stats registered with UUID: 352c04d7-6cc7-4726-9d7e-a7fa99cfb313","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433246669Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:54.433255688Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433403568Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433435556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433446586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433569827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433581857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"352c04d7-6cc7-4726-9d7e-a7fa99cfb313 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433598716Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433607785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"0","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433617474Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"1","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433627044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"2","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433636593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433646552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433656362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41354)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433671481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64539)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.43368412Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57085)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.433696219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434300079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"1","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434384864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"2","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434413222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"0","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434576691Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434641247Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[0]R flush_numbers[0..12]: [209, 212, 219, 221, 221, 208, 208, 216, 216, 216, 209, 216]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434688334Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 127, 127, 122, 122, 125, 125, 125, 122, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434700003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434711362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[1]R flush_numbers[0..12]: [209, 212, 219, 221, 221, 208, 208, 216, 216, 216, 209, 216]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434722312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 127, 127, 122, 122, 125, 125, 125, 122, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434732921Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.43474399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[2]R flush_numbers[0..12]: [209, 212, 219, 221, 221, 208, 208, 216, 216, 216, 209, 216]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434755039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 127, 127, 122, 122, 125, 125, 125, 122, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434768489Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434779748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434789297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434799157Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434808486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434817805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"":"downstairs","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434827695Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434836534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"0","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434846393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"1","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434856043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"client":"2","":"downstairs","session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434865952Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434875052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"352c04d7-6cc7-4726-9d7e-a7fa99cfb313 is now active with session: d673197d-2513-4307-aa47-b528b161213d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434885011Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.43489891Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"session_id":"d673197d-2513-4307-aa47-b528b161213d"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434908129Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Activated sub_volume 352c04d7-6cc7-4726-9d7e-a7fa99cfb313","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434916259Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.434925748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.435162993Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} 01/30 Write block 5 len 16384 data: 11 13 13 13 02/30 Write block 287 len 8192 data: 10 13 03/30 Read block 223 len 12288 04/30 Read block 246 len 8192 05/30 Read block 124 len 32768 06/30 Write block 468 len 4096 data: 7 07/30 Read block 595 len 16384 08/30 Write block 183 len 20480 data: 9 11 11 11 13 09/30 Read block 226 len 28672 10/30 Read block 480 len 36864 11/30 Read block 433 len 4096 12/30 Read block 203 len 8192 13/30 Flush 14/30 Write block 3 len 40960 data: 8 9 12 14 14 14 9 11 9 9 15/30 Write block 463 len 28672 data: 8 7 9 6 8 8 9 16/30 Write block 549 len 4096 data: 13 17/30 Write block 454 len 8192 data: 11 10 18/30 Write block 269 len 12288 data: 11 11 10 19/30 Flush 20/30 Write block 593 len 28672 data: 9 11 9 8 8 8 6 21/30 Write block 472 len 20480 data: 12 10 8 9 9 22/30 Write block 223 len 36864 data: 6 5 5 6 8 8 9 8 7 23/30 Write block 166 len 24576 data: 9 8 7 8 8 10 24/30 Write block 189 len 16384 data: 11 11 5 7 25/30 Read block 400 len 32768 26/30 Read block 156 len 32768 27/30 Write block 436 len 20480 data: 10 10 12 9 8 28/30 Read block 4 len 8192 29/30 Write block 122 len 20480 data: 12 13 13 12 12 30/30 Write block 545 len 40960 data: 12 10 13 13 14 12 10 8 9 9 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 2 2 28 Done 6 28 28 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:54.442742744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:54.442915563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1651,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 128 128 209 223 223 T F F <--- 1 020-039 124 124 124 212 212 212 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 127 127 127 221 221 221 F F F 4 080-099 127 127 127 221 221 221 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 122 122 122 208 208 208 F T T <--- 7 140-159 125 125 125 216 216 216 F F F 8 160-179 125 125 125 216 216 216 F T T <--- 9 180-199 125 128 128 216 222 222 F T T <--- 10 200-219 122 122 122 209 209 209 F F F 11 220-239 125 125 125 216 216 216 F T T <--- 12 240-259 125 125 125 216 216 216 F F F 13 260-279 126 128 128 218 223 223 F F F <--- 14 280-299 127 128 128 221 222 222 T F F <--- 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 126 126 126 217 217 217 F F F 19 380-399 126 126 126 217 217 217 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 127 127 127 221 221 221 F T T <--- 22 440-459 121 128 128 204 223 223 F T T <--- 23 460-479 127 128 128 221 223 223 T T T <--- 24 480-499 124 124 124 215 215 215 F F F 25 500-519 124 124 124 215 215 215 F F F 26 520-539 126 126 126 217 217 217 F F F 27 540-559 122 128 128 205 223 223 F T T <--- 28 560-579 122 122 122 209 209 209 F F F 29 580-599 104 104 104 173 173 173 F T T <--- Max gen: 128, Max flush: 223 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 223 223 223 F F F 1 020-039 124 124 124 212 212 212 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 127 127 127 221 221 221 F F F 4 080-099 127 127 127 221 221 221 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 129 129 129 224 224 224 F F F 7 140-159 125 125 125 216 216 216 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 129 129 129 224 224 224 F F F 10 200-219 122 122 122 209 209 209 F F F 11 220-239 129 129 129 224 224 224 F F F 12 240-259 125 125 125 216 216 216 F F F 13 260-279 128 128 128 223 223 223 F F F 14 280-299 128 128 128 222 222 222 F F F 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 126 126 126 217 217 217 F F F 19 380-399 126 126 126 217 217 217 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 129 129 129 224 224 224 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 124 124 124 215 215 215 F F F 25 500-519 124 124 124 215 215 215 F F F 26 520-539 126 126 126 217 217 217 F F F 27 540-559 129 129 129 224 224 224 F F F 28 560-579 122 122 122 209 209 209 F F F 29 580-599 129 129 129 224 224 224 F F F Max gen: 129, Max flush: 224 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.706218984Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.714676587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.714963329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.714987627Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.714996506Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715008226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Crucible f8411c16-e3be-43e9-82ca-2f317d968a60 has session id: 4949d157-2169-4773-a513-7db0c5b3a483","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715348983Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Upstairs opts: Upstairs UUID: f8411c16-e3be-43e9-82ca-2f317d968a60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715379041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Crucible stats registered with UUID: f8411c16-e3be-43e9-82ca-2f317d968a60","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715389841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:58.71539908Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715534621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.71555141Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715564359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715719549Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715736008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"f8411c16-e3be-43e9-82ca-2f317d968a60 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715768046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715777535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"0","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715787194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"1","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715796774Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"2","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715806383Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715816412Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715831111Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"ds_connection connected from Ok(127.0.0.1:44920)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715887418Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"ds_connection connected from Ok(127.0.0.1:55127)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715913026Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"ds_connection connected from Ok(127.0.0.1:45640)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.715924385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.716599171Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"0","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.716649288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"2","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.716786399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"1","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.71691388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.716937919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[0]R flush_numbers[0..12]: [223, 212, 219, 221, 221, 208, 224, 216, 224, 224, 209, 224]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717078179Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[0]R generation[0..12]: [128, 124, 126, 127, 127, 122, 129, 125, 129, 129, 122, 129]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717090329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717101508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[1]R flush_numbers[0..12]: [223, 212, 219, 221, 221, 208, 224, 216, 224, 224, 209, 224]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717112477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[1]R generation[0..12]: [128, 124, 126, 127, 127, 122, 129, 125, 129, 129, 122, 129]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717123367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717134516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[2]R flush_numbers[0..12]: [223, 212, 219, 221, 221, 208, 224, 216, 224, 224, 209, 224]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717145475Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[2]R generation[0..12]: [128, 124, 126, 127, 127, 122, 129, 125, 129, 129, 122, 129]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717156054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717167374Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717176953Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717186942Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717196312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717208141Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"":"downstairs","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.71721802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.71722698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"0","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717236829Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"1","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717246878Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"client":"2","":"downstairs","session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717257088Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717266357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"f8411c16-e3be-43e9-82ca-2f317d968a60 is now active with session: 4949d157-2169-4773-a513-7db0c5b3a483","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717276426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717285236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"session_id":"4949d157-2169-4773-a513-7db0c5b3a483"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717294495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Activated sub_volume f8411c16-e3be-43e9-82ca-2f317d968a60","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717302475Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717316084Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.717566197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"task":"crutest"} 01/30 Write block 362 len 32768 data: 10 13 14 12 16 15 16 13 02/30 Write block 87 len 32768 data: 9 10 11 12 12 10 8 8 03/30 Write block 510 len 20480 data: 9 7 7 7 8 04/30 Write block 186 len 32768 data: 12 14 14 12 12 6 8 8 05/30 Write block 391 len 8192 data: 12 14 06/30 Read block 115 len 20480 07/30 Read block 515 len 32768 08/30 Read block 519 len 12288 09/30 Write block 67 len 8192 data: 7 8 10/30 Write block 289 len 20480 data: 12 11 11 8 7 11/30 Write block 210 len 4096 data: 13 12/30 Write block 224 len 32768 data: 6 6 7 9 9 10 9 8 13/30 Flush 14/30 Write block 578 len 12288 data: 8 5 6 15/30 Write block 482 len 32768 data: 13 11 11 11 9 9 12 12 16/30 Read block 225 len 16384 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 227 len 28672 data: 10 10 11 10 9 6 6 21/30 Read block 210 len 4096 22/30 Read block 151 len 4096 23/30 Flush 24/30 Write block 140 len 20480 data: 8 8 12 11 11 25/30 Read block 288 len 4096 26/30 Write block 536 len 40960 data: 13 14 14 12 13 13 12 11 11 13 27/30 Write block 553 len 28672 data: 10 10 11 12 11 12 12 28/30 Write block 568 len 36864 data: 11 12 10 9 7 5 7 8 10 29/30 Write block 443 len 32768 data: 9 9 11 11 13 15 15 12 30/30 Read block 60 len 40960 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:32:58.725536373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:32:58.725718001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1659,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 223 223 223 F F F 1 020-039 124 124 124 212 212 212 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 130 127 130 225 221 225 F F F <--- 4 080-099 130 127 130 225 221 225 F F F <--- 5 100-119 122 122 122 208 208 208 F F F 6 120-139 129 129 129 224 224 224 F F F 7 140-159 125 125 125 216 216 216 T F T <--- 8 160-179 129 129 129 224 224 224 F F F 9 180-199 130 129 130 225 224 225 F F F <--- 10 200-219 130 122 130 225 209 225 F F F <--- 11 220-239 130 129 130 229 224 229 F F F <--- 12 240-259 125 125 125 216 216 216 F F F 13 260-279 128 128 128 223 223 223 F F F 14 280-299 130 128 130 225 222 225 F F F <--- 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 130 126 130 225 217 225 F T F <--- 19 380-399 130 126 130 225 217 225 F F F <--- 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 129 129 129 224 224 224 T F T <--- 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 124 130 226 215 226 F F F <--- 25 500-519 130 124 130 225 215 225 F F F <--- 26 520-539 126 126 126 217 217 217 T F T <--- 27 540-559 129 129 129 224 224 224 T F T <--- 28 560-579 130 122 130 226 209 226 T F T <--- 29 580-599 130 129 130 226 224 226 F F F <--- Max gen: 130, Max flush: 229 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 223 223 223 F F F 1 020-039 124 124 124 212 212 212 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 130 130 130 225 225 225 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 129 129 129 224 224 224 F F F 7 140-159 131 131 131 230 230 230 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 130 130 130 225 225 225 F F F 10 200-219 130 130 130 225 225 225 F F F 11 220-239 130 130 130 229 229 229 F F F 12 240-259 125 125 125 216 216 216 F F F 13 260-279 128 128 128 223 223 223 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 130 130 130 225 225 225 F F F 19 380-399 130 130 130 225 225 225 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 131 131 131 230 230 230 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 130 130 130 225 225 225 F F F 26 520-539 131 131 131 230 230 230 F F F 27 540-559 131 131 131 230 230 230 F F F 28 560-579 131 131 131 230 230 230 F F F 29 580-599 130 130 130 226 226 226 F F F Max gen: 131, Max flush: 230 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:02.988529143Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:02.995762667Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:02.995802634Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003107704Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003383386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003412764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003421583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003433162Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Crucible dab1dcdd-1f87-4e53-a6f6-6bd22482758c has session id: 2bf6ae27-7231-4143-9376-784ff36a26c0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003800998Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Upstairs opts: Upstairs UUID: dab1dcdd-1f87-4e53-a6f6-6bd22482758c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003826177Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Crucible stats registered with UUID: dab1dcdd-1f87-4e53-a6f6-6bd22482758c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003836796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:03.003845955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.003990256Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004013864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004024184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004191892Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004215721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"dab1dcdd-1f87-4e53-a6f6-6bd22482758c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.00423301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004245079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"0","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004254738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"1","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004264598Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"2","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004274227Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004284176Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004294076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:65244)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004390759Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:65264)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004419578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59022)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.004431307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005065345Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"1","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005091863Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"2","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005162389Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"0","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005466759Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005513456Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[0]R flush_numbers[0..12]: [223, 212, 219, 225, 225, 208, 224, 230, 224, 225, 225, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005548323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[0]R generation[0..12]: [128, 124, 126, 130, 130, 122, 129, 131, 129, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005563522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005574812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[1]R flush_numbers[0..12]: [223, 212, 219, 225, 225, 208, 224, 230, 224, 225, 225, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005585531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[1]R generation[0..12]: [128, 124, 126, 130, 130, 122, 129, 131, 129, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.00559621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005607459Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[2]R flush_numbers[0..12]: [223, 212, 219, 225, 225, 208, 224, 230, 224, 225, 225, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005618259Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[2]R generation[0..12]: [128, 124, 126, 130, 130, 122, 129, 131, 129, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005628778Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005639747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005649297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005659076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005668375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005677865Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"":"downstairs","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005687534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005696324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"0","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005707813Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"1","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005717612Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"client":"2","":"downstairs","session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005727302Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005736501Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"dab1dcdd-1f87-4e53-a6f6-6bd22482758c is now active with session: 2bf6ae27-7231-4143-9376-784ff36a26c0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.00574644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.0057551Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"session_id":"2bf6ae27-7231-4143-9376-784ff36a26c0"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005764629Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Activated sub_volume dab1dcdd-1f87-4e53-a6f6-6bd22482758c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005772719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.005780868Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.006083338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} 01/30 Write block 30 len 40960 data: 7 8 8 7 7 6 6 9 9 7 02/30 Read block 182 len 4096 03/30 Flush 04/30 Write block 253 len 24576 data: 9 8 6 7 6 6 05/30 Write block 567 len 8192 data: 13 12 06/30 Write block 452 len 28672 data: 10 11 12 11 10 8 8 07/30 Read block 461 len 36864 08/30 Read block 210 len 28672 09/30 Write block 509 len 36864 data: 7 10 8 8 8 9 12 14 13 10/30 Flush 11/30 Write block 214 len 8192 data: 18 20 12/30 Write block 186 len 8192 data: 13 15 13/30 Flush 14/30 Write block 295 len 36864 data: 8 7 7 7 9 10 9 10 11 15/30 Write block 523 len 8192 data: 9 8 16/30 Read block 296 len 40960 17/30 Read block 46 len 16384 18/30 Write block 123 len 40960 data: 14 14 13 13 11 11 11 9 12 11 19/30 Write block 545 len 12288 data: 14 11 14 20/30 Read block 301 len 36864 21/30 Read block 543 len 20480 22/30 Read block 472 len 4096 23/30 Read block 208 len 8192 24/30 Read block 366 len 12288 25/30 Read block 63 len 28672 26/30 Read block 586 len 12288 27/30 Write block 513 len 16384 data: 9 10 13 15 28/30 Write block 147 len 32768 data: 16 16 20 18 18 15 13 15 29/30 Write block 81 len 12288 data: 7 7 6 30/30 Read block 218 len 28672 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 3 Sent Sent Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 0 31 Done 1 28 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:03.01487907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:03.015081096Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1667,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 223 223 223 F F F 1 020-039 124 132 132 212 231 231 T F F <--- 2 040-059 126 126 126 219 219 219 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 130 130 130 225 225 225 F T T <--- 5 100-119 122 122 122 208 208 208 F F F 6 120-139 129 129 129 224 224 224 F T T <--- 7 140-159 131 131 131 230 230 230 F T T <--- 8 160-179 129 129 129 224 224 224 F F F 9 180-199 130 132 132 225 233 233 F F F <--- 10 200-219 130 132 132 225 233 233 F F F <--- 11 220-239 130 130 130 229 229 229 F F F 12 240-259 125 132 132 216 232 232 F F F <--- 13 260-279 128 128 128 223 223 223 F F F 14 280-299 130 130 130 225 225 225 F T T <--- 15 300-319 126 126 126 218 218 218 F T T <--- 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 130 130 130 225 225 225 F F F 19 380-399 130 130 130 225 225 225 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 131 132 132 230 232 232 F F F <--- 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 130 132 132 225 232 232 F T T <--- 26 520-539 131 131 131 230 230 230 F T T <--- 27 540-559 131 131 131 230 230 230 F T T <--- 28 560-579 131 132 132 230 232 232 F F F <--- 29 580-599 130 130 130 226 226 226 F F F Max gen: 132, Max flush: 233 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 223 223 223 F F F 1 020-039 132 132 132 231 231 231 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 133 133 133 234 234 234 F F F 5 100-119 122 122 122 208 208 208 F F F 6 120-139 133 133 133 234 234 234 F F F 7 140-159 133 133 133 234 234 234 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 132 132 132 233 233 233 F F F 10 200-219 132 132 132 233 233 233 F F F 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 128 128 128 223 223 223 F F F 14 280-299 133 133 133 234 234 234 F F F 15 300-319 133 133 133 234 234 234 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 130 130 130 225 225 225 F F F 19 380-399 130 130 130 225 225 225 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 132 132 132 232 232 232 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 133 133 133 234 234 234 F F F 26 520-539 133 133 133 234 234 234 F F F 27 540-559 133 133 133 234 234 234 F F F 28 560-579 132 132 132 232 232 232 F F F 29 580-599 130 130 130 226 226 226 F F F Max gen: 133, Max flush: 234 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.281809661Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.29019122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.290466792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.29049158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.290500299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.290512079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Crucible 1659b14b-88a4-480b-abbe-b78c21ba53de has session id: abc8b298-9003-4525-beca-75934578b994","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.290814629Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Upstairs opts: Upstairs UUID: 1659b14b-88a4-480b-abbe-b78c21ba53de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.290843207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Crucible stats registered with UUID: 1659b14b-88a4-480b-abbe-b78c21ba53de","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.290854086Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:07.290863415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291054003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291069392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291079361Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.29125376Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291267099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"1659b14b-88a4-480b-abbe-b78c21ba53de active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291284208Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291293137Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"0","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291302907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"1","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291312596Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"2","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291322235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291332555Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291345114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"ds_connection connected from Ok(127.0.0.1:56503)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.29139745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"ds_connection connected from Ok(127.0.0.1:59168)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.29140832Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"ds_connection connected from Ok(127.0.0.1:57675)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.291418679Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292061577Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"1","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292087485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"0","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292140142Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"2","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292387615Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292412234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[0]R flush_numbers[0..12]: [223, 231, 219, 225, 234, 208, 234, 234, 224, 233, 233, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292483859Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[0]R generation[0..12]: [128, 132, 126, 130, 133, 122, 133, 133, 129, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292495368Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292506537Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[1]R flush_numbers[0..12]: [223, 231, 219, 225, 234, 208, 234, 234, 224, 233, 233, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292517337Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[1]R generation[0..12]: [128, 132, 126, 130, 133, 122, 133, 133, 129, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292527956Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292541225Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[2]R flush_numbers[0..12]: [223, 231, 219, 225, 234, 208, 234, 234, 224, 233, 233, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292551954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[2]R generation[0..12]: [128, 132, 126, 130, 133, 122, 133, 133, 129, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292562494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292573483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292582882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292592702Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292602121Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292611581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"":"downstairs","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.29262141Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292630499Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"0","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292640459Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"1","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292650288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"client":"2","":"downstairs","session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292660277Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292669567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"1659b14b-88a4-480b-abbe-b78c21ba53de is now active with session: abc8b298-9003-4525-beca-75934578b994","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292679666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292689945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"session_id":"abc8b298-9003-4525-beca-75934578b994"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292699185Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Activated sub_volume 1659b14b-88a4-480b-abbe-b78c21ba53de","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292707284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.292715364Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.293010914Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"task":"crutest"} 01/30 Write block 113 len 12288 data: 9 9 8 02/30 Write block 135 len 40960 data: 12 11 13 10 10 9 9 13 12 12 03/30 Read block 191 len 24576 04/30 Read block 565 len 4096 05/30 Read block 140 len 24576 06/30 Read block 310 len 28672 07/30 Read block 508 len 20480 08/30 Write block 271 len 20480 data: 11 10 11 12 8 09/30 Read block 299 len 40960 10/30 Read block 110 len 28672 11/30 Write block 594 len 4096 data: 12 12/30 Write block 365 len 20480 data: 13 17 16 17 14 13/30 Write block 111 len 8192 data: 11 11 14/30 Read block 138 len 20480 15/30 Flush 16/30 Write block 285 len 12288 data: 11 11 11 17/30 Read block 88 len 4096 18/30 Write block 567 len 28672 data: 14 13 13 11 10 8 6 19/30 Read block 164 len 12288 20/30 Write block 24 len 32768 data: 9 8 9 7 8 8 8 9 21/30 Write block 185 len 4096 data: 12 22/30 Write block 1 len 40960 data: 7 7 9 10 13 15 15 15 10 12 23/30 Read block 117 len 28672 24/30 Read block 360 len 8192 25/30 Read block 38 len 12288 26/30 Flush 27/30 Write block 540 len 24576 data: 14 14 13 12 12 15 28/30 Read block 500 len 20480 29/30 Read block 472 len 40960 30/30 Write block 379 len 12288 data: 10 8 8 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:07.301245633Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:07.301427801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1675,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 128 236 236 223 F F F <--- 1 020-039 134 134 132 236 236 231 F F F <--- 2 040-059 126 126 126 219 219 219 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 133 133 133 234 234 234 F F F 5 100-119 134 134 122 235 235 208 F F T <--- 6 120-139 134 134 133 235 235 234 F F F <--- 7 140-159 134 134 133 235 235 234 F F F <--- 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 132 236 236 233 F F F <--- 10 200-219 132 132 132 233 233 233 F F F 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 134 134 128 235 235 223 F F F <--- 14 280-299 134 134 133 236 236 234 F F F <--- 15 300-319 133 133 133 234 234 234 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 134 134 130 235 235 225 T T F <--- 19 380-399 130 130 130 225 225 225 T T F <--- 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 132 132 132 232 232 232 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 133 133 133 234 234 234 F F F 26 520-539 133 133 133 234 234 234 F F F 27 540-559 133 133 133 234 234 234 T T F <--- 28 560-579 134 134 132 236 236 232 F F F <--- 29 580-599 134 134 130 235 235 226 F F F <--- Max gen: 134, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 134 134 134 236 236 236 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 133 133 133 234 234 234 F F F 5 100-119 134 134 134 235 235 235 F F F 6 120-139 134 134 134 235 235 235 F F F 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 132 132 132 233 233 233 F F F 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 134 134 134 235 235 235 F F F 14 280-299 134 134 134 236 236 236 F F F 15 300-319 133 133 133 234 234 234 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 135 135 135 237 237 237 F F F 19 380-399 135 135 135 237 237 237 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 F F F 22 440-459 132 132 132 232 232 232 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 130 130 226 226 226 F F F 25 500-519 133 133 133 234 234 234 F F F 26 520-539 133 133 133 234 234 234 F F F 27 540-559 135 135 135 237 237 237 F F F 28 560-579 134 134 134 236 236 236 F F F 29 580-599 134 134 134 235 235 235 F F F Max gen: 135, Max flush: 237 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.561236302Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.569686306Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570028333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570054302Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570062871Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.57007466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Crucible a33376d6-c7ec-4a17-8a18-32cfe9f09e3a has session id: 8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570397239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Upstairs opts: Upstairs UUID: a33376d6-c7ec-4a17-8a18-32cfe9f09e3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570426687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Crucible stats registered with UUID: a33376d6-c7ec-4a17-8a18-32cfe9f09e3a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570437846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:11.570447196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570595636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570624644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570635353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570779214Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570803462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"a33376d6-c7ec-4a17-8a18-32cfe9f09e3a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570817231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570826241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"0","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.57084196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"1","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570851699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"2","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570861268Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570871438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.570881397Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33696)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.571030867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:35907)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.571053676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42258)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.571064665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.571674525Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"0","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.571691984Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"1","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.571703113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"2","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572001873Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572011663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[0]R flush_numbers[0..12]: [236, 236, 219, 225, 234, 235, 235, 235, 224, 236, 233, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572025602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[0]R generation[0..12]: [134, 134, 126, 130, 133, 134, 134, 134, 129, 134, 132, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572036251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.57204732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[1]R flush_numbers[0..12]: [236, 236, 219, 225, 234, 235, 235, 235, 224, 236, 233, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.57205844Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[1]R generation[0..12]: [134, 134, 126, 130, 133, 134, 134, 134, 129, 134, 132, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572068989Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572080108Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[2]R flush_numbers[0..12]: [236, 236, 219, 225, 234, 235, 235, 235, 224, 236, 233, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572090718Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[2]R generation[0..12]: [134, 134, 126, 130, 133, 134, 134, 134, 129, 134, 132, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572101167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572112016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572121486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572133175Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572142554Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572151954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"":"downstairs","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572161683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572170502Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"0","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572180102Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"1","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572189741Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"client":"2","":"downstairs","session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.57219961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.57220874Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"a33376d6-c7ec-4a17-8a18-32cfe9f09e3a is now active with session: 8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572218949Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572227719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"session_id":"8851cd4a-85ca-4d0a-ad9d-a934c3eb83f8"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572236828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Activated sub_volume a33376d6-c7ec-4a17-8a18-32cfe9f09e3a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572244817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572252887Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.572585315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"task":"crutest"} 01/30 Write block 555 len 24576 data: 12 13 12 13 13 12 02/30 Read block 517 len 20480 03/30 Flush 04/30 Write block 365 len 16384 data: 14 18 17 18 05/30 Write block 530 len 28672 data: 11 11 11 10 9 11 14 06/30 Read block 480 len 16384 07/30 Read block 556 len 20480 08/30 Flush 09/30 Write block 540 len 16384 data: 15 15 14 13 10/30 Read block 441 len 4096 11/30 Read block 479 len 12288 12/30 Read block 545 len 12288 13/30 Write block 590 len 36864 data: 10 10 10 10 13 10 9 9 9 14/30 Write block 132 len 24576 data: 12 11 14 13 12 14 15/30 Read block 363 len 16384 16/30 Flush 17/30 Write block 108 len 40960 data: 10 9 10 12 12 10 10 9 9 9 18/30 Read block 187 len 8192 19/30 Flush 20/30 Write block 208 len 36864 data: 15 14 14 19 19 17 19 21 19 21/30 Flush 22/30 Write block 67 len 40960 data: 8 9 12 13 14 15 14 14 15 14 23/30 Read block 341 len 40960 24/30 Read block 139 len 8192 25/30 Flush 26/30 Write block 201 len 36864 data: 10 10 13 14 13 14 12 16 15 27/30 Write block 421 len 16384 data: 11 14 10 11 28/30 Write block 494 len 4096 data: 16 29/30 Write block 305 len 8192 data: 11 11 30/30 Read block 174 len 12288 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:11.582057742Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:11.58224593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1683,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 134 134 134 236 236 236 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 136 130 136 243 225 243 F F F <--- 4 080-099 133 133 133 234 234 234 F F F 5 100-119 136 134 136 241 235 241 F F F <--- 6 120-139 136 134 136 240 235 240 F F F <--- 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 136 132 136 242 233 242 T F T <--- 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 134 134 134 235 235 235 F F F 14 280-299 134 134 134 236 236 236 F F F 15 300-319 133 133 133 234 234 234 T F T <--- 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 136 135 136 239 237 239 F F F <--- 19 380-399 135 135 135 237 237 237 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 129 129 129 224 224 224 T F T <--- 22 440-459 132 132 132 232 232 232 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 130 130 130 226 226 226 T F T <--- 25 500-519 133 133 133 234 234 234 F F F 26 520-539 136 133 136 239 234 239 F F F <--- 27 540-559 136 135 136 240 237 240 F T F <--- 28 560-579 136 134 136 238 236 238 F T F <--- 29 580-599 136 134 136 240 235 240 F F F <--- Max gen: 136, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 134 134 134 236 236 236 F F F 2 040-059 126 126 126 219 219 219 F F F 3 060-079 136 136 136 243 243 243 F F F 4 080-099 133 133 133 234 234 234 F F F 5 100-119 136 136 136 241 241 241 F F F 6 120-139 136 136 136 240 240 240 F F F 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 134 134 134 235 235 235 F F F 14 280-299 134 134 134 236 236 236 F F F 15 300-319 137 137 137 244 244 244 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 126 126 126 218 218 218 F F F 18 360-379 136 136 136 239 239 239 F F F 19 380-399 135 135 135 237 237 237 F F F 20 400-419 127 127 127 221 221 221 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 132 132 132 232 232 232 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 133 133 133 234 234 234 F F F 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 136 136 136 238 238 238 F F F 29 580-599 136 136 136 240 240 240 F F F Max gen: 137, Max flush: 244 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.83688035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845097589Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845377851Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845404049Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845412399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845428248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Crucible 798fb00a-7b9b-40a7-b7d1-9dd1edba1b36 has session id: bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845765215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Upstairs opts: Upstairs UUID: 798fb00a-7b9b-40a7-b7d1-9dd1edba1b36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845795043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Crucible stats registered with UUID: 798fb00a-7b9b-40a7-b7d1-9dd1edba1b36","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845806273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:15.845815482Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845948763Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845979101Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.845990161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846133221Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.84614515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"798fb00a-7b9b-40a7-b7d1-9dd1edba1b36 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846162259Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846173739Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"0","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846183488Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"1","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846193167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"2","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846202777Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846212736Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846222665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50246)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846272572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43848)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846283531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37461)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846293781Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.846994315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"1","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847011403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"0","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847073799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"2","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847305314Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847314744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[0]R flush_numbers[0..12]: [236, 236, 219, 243, 234, 241, 240, 235, 224, 236, 244, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847397698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[0]R generation[0..12]: [134, 134, 126, 136, 133, 136, 136, 134, 129, 134, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847411517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847422596Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[1]R flush_numbers[0..12]: [236, 236, 219, 243, 234, 241, 240, 235, 224, 236, 244, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847433426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[1]R generation[0..12]: [134, 134, 126, 136, 133, 136, 136, 134, 129, 134, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847444055Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847455094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[2]R flush_numbers[0..12]: [236, 236, 219, 243, 234, 241, 240, 235, 224, 236, 244, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847465794Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[2]R generation[0..12]: [134, 134, 126, 136, 133, 136, 136, 134, 129, 134, 137, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847476383Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847487262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847496812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847506561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.84751593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.84752545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"":"downstairs","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847535379Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847544238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"0","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847555758Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"1","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847565547Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"client":"2","":"downstairs","session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847575626Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847585016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"798fb00a-7b9b-40a7-b7d1-9dd1edba1b36 is now active with session: bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847595065Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847603785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"session_id":"bcb14dfe-9d6d-4dc3-8b4e-104d8de88ae2"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847613054Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Activated sub_volume 798fb00a-7b9b-40a7-b7d1-9dd1edba1b36","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-18T03:33:15.847621033Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847633153Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.847886346Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"task":"crutest"} 01/30 Write block 47 len 16384 data: 9 8 7 8 02/30 Write block 115 len 20480 data: 10 10 10 10 11 03/30 Read block 458 len 28672 04/30 Read block 545 len 36864 05/30 Read block 78 len 20480 06/30 Write block 585 len 28672 data: 7 8 8 11 10 11 11 07/30 Read block 584 len 28672 08/30 Read block 465 len 12288 09/30 Write block 54 len 12288 data: 9 6 7 10/30 Write block 346 len 20480 data: 15 16 14 12 13 11/30 Read block 24 len 24576 12/30 Read block 534 len 28672 13/30 Flush 14/30 Write block 56 len 24576 data: 8 7 5 8 8 10 15/30 Flush 16/30 Write block 212 len 4096 data: 20 17/30 Read block 28 len 40960 18/30 Read block 543 len 4096 19/30 Read block 140 len 28672 20/30 Write block 319 len 16384 data: 13 12 12 13 21/30 Write block 394 len 32768 data: 13 9 7 7 7 8 8 7 22/30 Read block 504 len 4096 23/30 Write block 586 len 12288 data: 9 9 12 24/30 Write block 486 len 16384 data: 10 10 13 13 25/30 Read block 362 len 32768 26/30 Write block 509 len 36864 data: 8 11 9 9 10 11 14 16 14 27/30 Write block 377 len 40960 data: 8 10 11 9 9 8 9 8 9 10 28/30 Write block 451 len 32768 data: 11 11 12 13 12 11 9 9 29/30 Write block 581 len 12288 data: 5 5 6 30/30 Read block 589 len 12288 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Write 5 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:15.855812625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:15.855991353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1691,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 134 134 134 236 236 236 F F F 2 040-059 138 138 126 246 246 219 F F F <--- 3 060-079 138 138 136 246 246 243 F F F <--- 4 080-099 133 133 133 234 234 234 F F F 5 100-119 138 138 136 245 245 241 F F F <--- 6 120-139 136 136 136 240 240 240 F F F 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 137 137 137 244 244 244 T T F <--- 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 134 134 134 235 235 235 F F F 14 280-299 134 134 134 236 236 236 F F F 15 300-319 137 137 137 244 244 244 T T F <--- 16 320-339 126 126 126 220 220 220 T T F <--- 17 340-359 138 138 126 245 245 218 F F F <--- 18 360-379 136 136 136 239 239 239 T T F <--- 19 380-399 135 135 135 237 237 237 T T F <--- 20 400-419 127 127 127 221 221 221 T T F <--- 21 420-439 137 137 137 244 244 244 F F F 22 440-459 132 132 132 232 232 232 T T F <--- 23 460-479 129 129 129 224 224 224 F F F 24 480-499 137 137 137 244 244 244 T T F <--- 25 500-519 133 133 133 234 234 234 T T F <--- 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 136 136 136 238 238 238 F F F 29 580-599 138 138 136 245 245 240 T T F <--- Max gen: 138, Max flush: 246 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 134 134 134 236 236 236 F F F 2 040-059 138 138 138 246 246 246 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 133 133 133 234 234 234 F F F 5 100-119 138 138 138 245 245 245 F F F 6 120-139 136 136 136 240 240 240 F F F 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 139 139 139 247 247 247 F F F 11 220-239 130 130 130 229 229 229 F F F 12 240-259 132 132 132 232 232 232 F F F 13 260-279 134 134 134 235 235 235 F F F 14 280-299 134 134 134 236 236 236 F F F 15 300-319 139 139 139 247 247 247 F F F 16 320-339 139 139 139 247 247 247 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 139 139 139 247 247 247 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 139 139 139 247 247 247 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 139 139 139 247 247 247 F F F 25 500-519 139 139 139 247 247 247 F F F 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 136 136 136 238 238 238 F F F 29 580-599 139 139 139 247 247 247 F F F Max gen: 139, Max flush: 247 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.116127242Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:20.123359057Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.123405484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.130568282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.130822476Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.130848904Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.130859863Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.130871682Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Crucible c2435806-8db4-43de-84bc-e733ce2d97ee has session id: 13817524-ec73-4c9a-b16a-7752109bac0a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131199081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Upstairs opts: Upstairs UUID: c2435806-8db4-43de-84bc-e733ce2d97ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131269946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Crucible stats registered with UUID: c2435806-8db4-43de-84bc-e733ce2d97ee","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131296805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:20.131320773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131409557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131437525Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131448445Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131588005Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131605134Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"c2435806-8db4-43de-84bc-e733ce2d97ee active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131628563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131638092Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"0","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131648041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"1","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131657851Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"2","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.1316698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131680119Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131690159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57962)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131745575Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37368)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131756644Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63086)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.131854158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.13242892Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"1","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132458548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"2","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132506375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"0","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132744609Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132765068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[0]R flush_numbers[0..12]: [236, 236, 246, 246, 234, 245, 240, 235, 224, 236, 247, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132841423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[0]R generation[0..12]: [134, 134, 138, 138, 133, 138, 136, 134, 129, 134, 139, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132853392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132864601Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[1]R flush_numbers[0..12]: [236, 236, 246, 246, 234, 245, 240, 235, 224, 236, 247, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132875391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[1]R generation[0..12]: [134, 134, 138, 138, 133, 138, 136, 134, 129, 134, 139, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.13288885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132900369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[2]R flush_numbers[0..12]: [236, 236, 246, 246, 234, 245, 240, 235, 224, 236, 247, 229]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132911138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[2]R generation[0..12]: [134, 134, 138, 138, 133, 138, 136, 134, 129, 134, 139, 130]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132921938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132932877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132942516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132952276Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132961705Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132971224Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"":"downstairs","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132981114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132989883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"0","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.132999783Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"1","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133009642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"client":"2","":"downstairs","session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133019491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133028681Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"c2435806-8db4-43de-84bc-e733ce2d97ee is now active with session: 13817524-ec73-4c9a-b16a-7752109bac0a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.13304047Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133049419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"session_id":"13817524-ec73-4c9a-b16a-7752109bac0a"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133058629Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Activated sub_volume c2435806-8db4-43de-84bc-e733ce2d97ee","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133066698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133074808Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.133394687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} 01/30 Flush 02/30 Write block 85 len 16384 data: 5 8 10 11 03/30 Read block 570 len 24576 04/30 Write block 298 len 4096 data: 8 05/30 Write block 111 len 8192 data: 13 13 06/30 Write block 445 len 40960 data: 12 12 14 16 16 13 12 12 13 14 07/30 Read block 407 len 32768 08/30 Write block 251 len 40960 data: 10 9 10 9 7 8 7 7 6 7 09/30 Read block 470 len 8192 10/30 Write block 215 len 24576 data: 22 20 15 13 10 8 11/30 Write block 107 len 16384 data: 10 11 10 11 12/30 Read block 407 len 24576 13/30 Read block 213 len 40960 14/30 Read block 330 len 12288 15/30 Read block 3 len 4096 16/30 Read block 120 len 28672 17/30 Write block 37 len 20480 data: 10 10 8 7 7 18/30 Read block 562 len 36864 19/30 Read block 477 len 12288 20/30 Write block 220 len 36864 data: 9 8 7 7 7 7 8 11 11 21/30 Read block 453 len 32768 22/30 Read block 454 len 28672 23/30 Write block 584 len 20480 data: 6 8 10 10 13 24/30 Read block 359 len 12288 25/30 Write block 34 len 36864 data: 8 7 7 11 11 9 8 8 9 26/30 Read block 310 len 4096 27/30 Read block 215 len 20480 28/30 Write block 571 len 8192 data: 11 9 29/30 Read block 93 len 28672 30/30 Read block 558 len 40960 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:20.142794088Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:20.142976346Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1699,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 134 134 134 236 236 236 F T T <--- 2 040-059 138 138 138 246 246 246 F T T <--- 3 060-079 138 138 138 246 246 246 F F F 4 080-099 133 133 133 234 234 234 F T T <--- 5 100-119 138 138 138 245 245 245 F T T <--- 6 120-139 136 136 136 240 240 240 F F F 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 139 139 139 247 247 247 F T T <--- 11 220-239 130 130 130 229 229 229 F T T <--- 12 240-259 132 132 132 232 232 232 F T T <--- 13 260-279 134 134 134 235 235 235 F T T <--- 14 280-299 134 134 134 236 236 236 F T T <--- 15 300-319 139 139 139 247 247 247 F F F 16 320-339 139 139 139 247 247 247 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 139 139 139 247 247 247 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 139 139 139 247 247 247 F T T <--- 23 460-479 129 129 129 224 224 224 F F F 24 480-499 139 139 139 247 247 247 F F F 25 500-519 139 139 139 247 247 247 F F F 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 136 136 136 238 238 238 F T T <--- 29 580-599 139 139 139 247 247 247 F T T <--- Max gen: 139, Max flush: 247 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 140 140 140 248 248 248 F F F 2 040-059 140 140 140 248 248 248 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 140 140 140 248 248 248 F F F 5 100-119 140 140 140 248 248 248 F F F 6 120-139 136 136 136 240 240 240 F F F 7 140-159 134 134 134 235 235 235 F F F 8 160-179 129 129 129 224 224 224 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 140 140 140 248 248 248 F F F 12 240-259 140 140 140 248 248 248 F F F 13 260-279 140 140 140 248 248 248 F F F 14 280-299 140 140 140 248 248 248 F F F 15 300-319 139 139 139 247 247 247 F F F 16 320-339 139 139 139 247 247 247 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 139 139 139 247 247 247 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 139 139 139 247 247 247 F F F 25 500-519 139 139 139 247 247 247 F F F 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 140 140 140 248 248 248 F F F 29 580-599 140 140 140 248 248 248 F F F Max gen: 140, Max flush: 248 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.406668941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415071308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415355099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415381758Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415390567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415406356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Crucible 83d3fbf0-1267-4936-9441-321b26006a2f has session id: b6c4d542-515e-4a6f-8d5c-06db87bfec1a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415742134Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Upstairs opts: Upstairs UUID: 83d3fbf0-1267-4936-9441-321b26006a2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415763493Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Crucible stats registered with UUID: 83d3fbf0-1267-4936-9441-321b26006a2f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415775812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:24.415784901Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.415933531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.41594857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.41595895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.41610321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416119549Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"83d3fbf0-1267-4936-9441-321b26006a2f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416136468Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416145317Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"0","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416155007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"1","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416164796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"2","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416174446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416184585Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416194484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57747)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416310017Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48326)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416367213Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60693)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416378582Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416921796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"0","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416933886Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"1","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.416944645Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"2","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417238726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417248285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[0]R flush_numbers[0..12]: [236, 248, 248, 246, 248, 248, 240, 235, 224, 236, 248, 248]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417291282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[0]R generation[0..12]: [134, 140, 140, 138, 140, 140, 136, 134, 129, 134, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417302571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417313721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[1]R flush_numbers[0..12]: [236, 248, 248, 246, 248, 248, 240, 235, 224, 236, 248, 248]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.41732429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[1]R generation[0..12]: [134, 140, 140, 138, 140, 140, 136, 134, 129, 134, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417335229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417346328Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[2]R flush_numbers[0..12]: [236, 248, 248, 246, 248, 248, 240, 235, 224, 236, 248, 248]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417356978Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[2]R generation[0..12]: [134, 140, 140, 138, 140, 140, 136, 134, 129, 134, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417367577Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417380466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417389906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417399755Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417409244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417418604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"":"downstairs","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417428653Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417437522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"0","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417447592Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"1","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417457471Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"client":"2","":"downstairs","session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.41746748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.41747676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"83d3fbf0-1267-4936-9441-321b26006a2f is now active with session: b6c4d542-515e-4a6f-8d5c-06db87bfec1a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417486819Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-18T03:33:24.417495419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"session_id":"b6c4d542-515e-4a6f-8d5c-06db87bfec1a"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417508508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Activated sub_volume 83d3fbf0-1267-4936-9441-321b26006a2f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417516637Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417524767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.417763761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"task":"crutest"} 01/30 Flush 02/30 Write block 108 len 12288 data: 12 11 12 03/30 Read block 409 len 36864 04/30 Read block 119 len 20480 05/30 Read block 108 len 40960 06/30 Read block 90 len 12288 07/30 Read block 204 len 20480 08/30 Read block 400 len 24576 09/30 Flush 10/30 Write block 235 len 36864 data: 10 10 11 10 12 11 9 11 10 11/30 Write block 320 len 12288 data: 13 13 14 12/30 Read block 57 len 28672 13/30 Read block 244 len 24576 14/30 Read block 457 len 28672 15/30 Write block 565 len 16384 data: 11 12 15 14 16/30 Read block 273 len 4096 17/30 Flush 18/30 Write block 148 len 28672 data: 17 21 19 19 16 14 16 19/30 Read block 562 len 36864 20/30 Read block 388 len 16384 21/30 Read block 265 len 16384 22/30 Flush 23/30 Flush 24/30 Write block 492 len 16384 data: 13 14 17 15 25/30 Flush 26/30 Write block 166 len 28672 data: 10 9 8 9 9 11 7 27/30 Write block 145 len 4096 data: 12 28/30 Write block 250 len 16384 data: 13 11 10 11 29/30 Write block 299 len 28672 data: 10 11 10 11 12 12 12 30/30 Read block 189 len 20480 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 29 1 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1000)) Some(JobId(1024)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:24.42659886Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:24.426782328Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1707,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 140 140 140 248 248 248 F F F 2 040-059 140 140 140 248 248 248 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 140 140 140 248 248 248 F F F 5 100-119 142 140 142 250 248 250 F T F <--- 6 120-139 136 136 136 240 240 240 F F F 7 140-159 142 134 142 252 235 252 T F T <--- 8 160-179 129 129 129 224 224 224 T F T <--- 9 180-199 134 134 134 236 236 236 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 142 140 142 251 248 251 F F F <--- 12 240-259 142 140 142 251 248 251 T F T <--- 13 260-279 140 140 140 248 248 248 F F F 14 280-299 140 140 140 248 248 248 T F T <--- 15 300-319 139 139 139 247 247 247 T F T <--- 16 320-339 142 139 142 251 247 251 F F F <--- 17 340-359 138 138 138 245 245 245 F F F 18 360-379 139 139 139 247 247 247 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 142 139 142 254 247 254 F F F <--- 25 500-519 139 139 139 247 247 247 F F F 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 142 140 142 251 248 251 F F F <--- 29 580-599 140 140 140 248 248 248 F F F Max gen: 142, Max flush: 254 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 140 140 140 248 248 248 F F F 2 040-059 140 140 140 248 248 248 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 140 140 140 248 248 248 F F F 5 100-119 142 142 142 250 250 250 F F F 6 120-139 136 136 136 240 240 240 F F F 7 140-159 143 143 143 255 255 255 F F F 8 160-179 143 143 143 255 255 255 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 143 143 143 255 255 255 F F F 13 260-279 140 140 140 248 248 248 F F F 14 280-299 143 143 143 255 255 255 F F F 15 300-319 143 143 143 255 255 255 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 139 139 139 247 247 247 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 129 129 129 224 224 224 F F F 24 480-499 142 142 142 254 254 254 F F F 25 500-519 139 139 139 247 247 247 F F F 26 520-539 136 136 136 239 239 239 F F F 27 540-559 136 136 136 240 240 240 F F F 28 560-579 142 142 142 251 251 251 F F F 29 580-599 140 140 140 248 248 248 F F F Max gen: 143, Max flush: 255 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.67639115Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.684770688Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685098907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685133915Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685143204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685155073Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Crucible 9e4be6cb-9920-4fe0-ba16-71a523a89e5a has session id: 3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685574846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Upstairs opts: Upstairs UUID: 9e4be6cb-9920-4fe0-ba16-71a523a89e5a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685604144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Crucible stats registered with UUID: 9e4be6cb-9920-4fe0-ba16-71a523a89e5a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685615443Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:28.685624872Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685773052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685802381Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.68581349Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685951351Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685975649Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"9e4be6cb-9920-4fe0-ba16-71a523a89e5a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.685989418Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686008737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"0","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686018616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"1","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686028456Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"2","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686037935Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686047994Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686057844Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:36854)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686106771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:53823)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.68611775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45057)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686131369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686702481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"2","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686769487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"0","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686816584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"1","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.686980003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.68703233Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[0]R flush_numbers[0..12]: [236, 248, 248, 246, 248, 250, 240, 255, 255, 236, 248, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687105865Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[0]R generation[0..12]: [134, 140, 140, 138, 140, 142, 136, 143, 143, 134, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687117284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687128473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[1]R flush_numbers[0..12]: [236, 248, 248, 246, 248, 250, 240, 255, 255, 236, 248, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687139323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[1]R generation[0..12]: [134, 140, 140, 138, 140, 142, 136, 143, 143, 134, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687150132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687161271Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[2]R flush_numbers[0..12]: [236, 248, 248, 246, 248, 250, 240, 255, 255, 236, 248, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.68717216Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[2]R generation[0..12]: [134, 140, 140, 138, 140, 142, 136, 143, 143, 134, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.6871829Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687194329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687204018Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687213828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687223097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687232596Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"":"downstairs","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687242346Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687251245Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"0","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687262465Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"1","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687272394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"client":"2","":"downstairs","session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687282273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687291493Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"9e4be6cb-9920-4fe0-ba16-71a523a89e5a is now active with session: 3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687301632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687310371Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"session_id":"3fc17dbc-04a2-4d2d-a96b-b0bbcc59f5dc"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687319531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Activated sub_volume 9e4be6cb-9920-4fe0-ba16-71a523a89e5a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.68732749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.6873357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.687655009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"task":"crutest"} 01/30 Write block 56 len 8192 data: 9 8 02/30 Read block 341 len 16384 03/30 Read block 328 len 20480 04/30 Read block 250 len 24576 05/30 Read block 231 len 12288 06/30 Read block 195 len 16384 07/30 Read block 315 len 16384 08/30 Flush 09/30 Write block 532 len 16384 data: 12 11 10 12 10/30 Read block 267 len 36864 11/30 Write block 174 len 20480 data: 10 9 9 11 10 12/30 Flush 13/30 Write block 364 len 24576 data: 15 15 19 18 19 15 14/30 Read block 106 len 20480 15/30 Flush 16/30 Write block 474 len 4096 data: 9 17/30 Read block 150 len 24576 18/30 Read block 227 len 28672 19/30 Read block 59 len 20480 20/30 Read block 312 len 28672 21/30 Read block 250 len 40960 22/30 Read block 305 len 12288 23/30 Read block 484 len 24576 24/30 Flush 25/30 Write block 289 len 12288 data: 13 12 12 26/30 Write block 60 len 4096 data: 9 27/30 Write block 573 len 16384 data: 7 8 9 11 28/30 Write block 558 len 20480 data: 14 14 13 13 12 29/30 Read block 92 len 36864 30/30 Write block 81 len 20480 data: 8 8 7 7 6 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:28.695881708Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:28.696088114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1715,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 140 140 140 248 248 248 F F F 2 040-059 144 140 144 256 248 256 F T F <--- 3 060-079 138 138 138 246 246 246 T F T <--- 4 080-099 140 140 140 248 248 248 T F T <--- 5 100-119 142 142 142 250 250 250 F F F 6 120-139 136 136 136 240 240 240 F F F 7 140-159 143 143 143 255 255 255 F F F 8 160-179 144 143 144 257 255 257 F F F <--- 9 180-199 134 134 134 236 236 236 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 143 143 143 255 255 255 F F F 13 260-279 140 140 140 248 248 248 F F F 14 280-299 143 143 143 255 255 255 T F T <--- 15 300-319 143 143 143 255 255 255 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 144 139 144 258 247 258 F F F <--- 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 144 129 144 259 224 259 F F F <--- 24 480-499 142 142 142 254 254 254 F F F 25 500-519 139 139 139 247 247 247 F F F 26 520-539 144 136 144 257 239 257 F F F <--- 27 540-559 136 136 136 240 240 240 T F T <--- 28 560-579 142 142 142 251 251 251 T F T <--- 29 580-599 140 140 140 248 248 248 F F F Max gen: 144, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 236 236 236 F F F 1 020-039 140 140 140 248 248 248 F F F 2 040-059 144 144 144 256 256 256 F F F 3 060-079 145 145 145 260 260 260 F F F 4 080-099 145 145 145 260 260 260 F F F 5 100-119 142 142 142 250 250 250 F F F 6 120-139 136 136 136 240 240 240 F F F 7 140-159 143 143 143 255 255 255 F F F 8 160-179 144 144 144 257 257 257 F F F 9 180-199 134 134 134 236 236 236 F F F 10 200-219 140 140 140 248 248 248 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 143 143 143 255 255 255 F F F 13 260-279 140 140 140 248 248 248 F F F 14 280-299 145 145 145 260 260 260 F F F 15 300-319 143 143 143 255 255 255 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 144 144 144 259 259 259 F F F 24 480-499 142 142 142 254 254 254 F F F 25 500-519 139 139 139 247 247 247 F F F 26 520-539 144 144 144 257 257 257 F F F 27 540-559 145 145 145 260 260 260 F F F 28 560-579 145 145 145 260 260 260 F F F 29 580-599 140 140 140 248 248 248 F F F Max gen: 145, Max flush: 260 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.944945304Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953233949Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953515931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953546359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953555108Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953566947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Crucible 3d32ec31-9cab-4611-b65e-e536f4979a8f has session id: cc42191f-7239-4693-873c-955601a86914","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953898745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Upstairs opts: Upstairs UUID: 3d32ec31-9cab-4611-b65e-e536f4979a8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953928353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Crucible stats registered with UUID: 3d32ec31-9cab-4611-b65e-e536f4979a8f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.953939283Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:32.953948562Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954079793Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954098522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954108772Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954293979Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954318388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"3d32ec31-9cab-4611-b65e-e536f4979a8f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954334687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954343696Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"0","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954380074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"1","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954402722Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"2","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954413332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954423691Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95443632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"ds_connection connected from Ok(127.0.0.1:37061)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954447209Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"ds_connection connected from Ok(127.0.0.1:47384)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954457849Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"ds_connection connected from Ok(127.0.0.1:50189)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.954468228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95504962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"1","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955071508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"0","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955169612Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"2","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95534343Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955359799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[0]R flush_numbers[0..12]: [236, 248, 256, 260, 260, 250, 240, 255, 257, 236, 248, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955436344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[0]R generation[0..12]: [134, 140, 144, 145, 145, 142, 136, 143, 144, 134, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955447883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955458893Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[1]R flush_numbers[0..12]: [236, 248, 256, 260, 260, 250, 240, 255, 257, 236, 248, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955469592Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[1]R generation[0..12]: [134, 140, 144, 145, 145, 142, 136, 143, 144, 134, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955480211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955491181Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[2]R flush_numbers[0..12]: [236, 248, 256, 260, 260, 250, 240, 255, 257, 236, 248, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95550477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[2]R generation[0..12]: [134, 140, 144, 145, 145, 142, 136, 143, 144, 134, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955515259Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955526378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955535858Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955545617Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955554836Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955564426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"":"downstairs","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955574215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955582955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"0","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955592594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"1","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955602313Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"client":"2","":"downstairs","session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955612103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955621222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"3d32ec31-9cab-4611-b65e-e536f4979a8f is now active with session: cc42191f-7239-4693-873c-955601a86914","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955631201Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955639981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"session_id":"cc42191f-7239-4693-873c-955601a86914"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95565061Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Activated sub_volume 3d32ec31-9cab-4611-b65e-e536f4979a8f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95565879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.955666899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.95596019Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"task":"crutest"} 01/30 Write block 8 len 20480 data: 16 11 13 10 10 02/30 Read block 554 len 32768 03/30 Read block 162 len 32768 04/30 Read block 141 len 40960 05/30 Write block 91 len 12288 data: 13 11 9 06/30 Read block 122 len 16384 07/30 Write block 124 len 12288 data: 15 14 14 08/30 Write block 22 len 32768 data: 12 11 10 9 10 8 9 9 09/30 Flush 10/30 Write block 528 len 4096 data: 8 11/30 Read block 39 len 32768 12/30 Read block 521 len 8192 13/30 Read block 426 len 28672 14/30 Write block 316 len 4096 data: 10 15/30 Read block 381 len 20480 16/30 Write block 514 len 16384 data: 12 15 17 15 17/30 Read block 366 len 4096 18/30 Flush 19/30 Write block 122 len 24576 data: 13 15 16 15 15 12 20/30 Read block 324 len 36864 21/30 Write block 174 len 28672 data: 11 10 10 12 11 11 12 22/30 Read block 183 len 36864 23/30 Read block 457 len 12288 24/30 Write block 212 len 4096 data: 21 25/30 Write block 588 len 36864 data: 14 11 12 12 11 11 14 11 10 26/30 Write block 115 len 32768 data: 11 11 11 11 12 13 11 14 27/30 Read block 176 len 24576 28/30 Write block 179 len 40960 data: 12 13 12 12 10 12 13 14 16 15 29/30 Write block 273 len 8192 data: 12 13 30/30 Read block 196 len 12288 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 10 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:32.964734273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:32.964913331Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1723,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 134 261 261 236 F F T <--- 1 020-039 146 146 140 261 261 248 F F F <--- 2 040-059 144 144 144 256 256 256 F F F 3 060-079 145 145 145 260 260 260 F F F 4 080-099 146 146 145 261 261 260 F F F <--- 5 100-119 142 142 142 250 250 250 T T F <--- 6 120-139 146 146 136 261 261 240 T T F <--- 7 140-159 143 143 143 255 255 255 F F F 8 160-179 144 144 144 257 257 257 T T F <--- 9 180-199 134 134 134 236 236 236 T T F <--- 10 200-219 140 140 140 248 248 248 T T F <--- 11 220-239 142 142 142 251 251 251 F F F 12 240-259 143 143 143 255 255 255 F F F 13 260-279 140 140 140 248 248 248 T T F <--- 14 280-299 145 145 145 260 260 260 F F F 15 300-319 146 146 143 262 262 255 F F F <--- 16 320-339 142 142 142 251 251 251 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 144 144 144 259 259 259 F F F 24 480-499 142 142 142 254 254 254 F F F 25 500-519 146 146 139 262 262 247 F F F <--- 26 520-539 146 146 144 262 262 257 F F F <--- 27 540-559 145 145 145 260 260 260 F F F 28 560-579 145 145 145 260 260 260 F F F 29 580-599 140 140 140 248 248 248 T T F <--- Max gen: 146, Max flush: 262 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 261 261 261 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 144 144 144 256 256 256 F F F 3 060-079 145 145 145 260 260 260 F F F 4 080-099 146 146 146 261 261 261 F F F 5 100-119 147 147 147 263 263 263 F F F 6 120-139 147 147 147 263 263 263 F F F 7 140-159 143 143 143 255 255 255 F F F 8 160-179 147 147 147 263 263 263 F F F 9 180-199 147 147 147 263 263 263 F F F 10 200-219 147 147 147 263 263 263 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 143 143 143 255 255 255 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 145 145 145 260 260 260 F F F 15 300-319 146 146 146 262 262 262 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 138 138 138 245 245 245 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 144 144 144 259 259 259 F F F 24 480-499 142 142 142 254 254 254 F F F 25 500-519 146 146 146 262 262 262 F F F 26 520-539 146 146 146 262 262 262 F F F 27 540-559 145 145 145 260 260 260 F F F 28 560-579 145 145 145 260 260 260 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 147, Max flush: 263 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.226066112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.234366016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.234656717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.234707214Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.234733642Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.234746581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Crucible 326baef2-ac5c-47cd-9fab-27ce440cb185 has session id: 0337f90b-37e7-459a-ab9a-1368af4968bd","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.23507481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Upstairs opts: Upstairs UUID: 326baef2-ac5c-47cd-9fab-27ce440cb185, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235104138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Crucible stats registered with UUID: 326baef2-ac5c-47cd-9fab-27ce440cb185","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235115657Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:37.235125007Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235262907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235278476Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235288756Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235446555Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235461454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"326baef2-ac5c-47cd-9fab-27ce440cb185 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235494932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235504362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"0","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235514011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"1","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.23552345Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"2","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.23553309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235543079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235553148Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46433)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235604945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44282)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235615954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58940)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.235631633Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236343206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"0","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236370925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"2","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236382094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"1","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236684084Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236697323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[0]R flush_numbers[0..12]: [261, 261, 256, 260, 261, 263, 263, 255, 263, 263, 263, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236707292Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[0]R generation[0..12]: [146, 146, 144, 145, 146, 147, 147, 143, 147, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236718152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236729131Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[1]R flush_numbers[0..12]: [261, 261, 256, 260, 261, 263, 263, 255, 263, 263, 263, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.23673985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[1]R generation[0..12]: [146, 146, 144, 145, 146, 147, 147, 143, 147, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.23675051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236761539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[2]R flush_numbers[0..12]: [261, 261, 256, 260, 261, 263, 263, 255, 263, 263, 263, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236772128Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[2]R generation[0..12]: [146, 146, 144, 145, 146, 147, 147, 143, 147, 147, 147, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236782808Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236793837Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236803156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236815145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236824685Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236834234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"":"downstairs","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236844423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236853163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"0","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236863122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"1","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236878641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"client":"2","":"downstairs","session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236888771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.23689808Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"326baef2-ac5c-47cd-9fab-27ce440cb185 is now active with session: 0337f90b-37e7-459a-ab9a-1368af4968bd","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236908279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236917159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"session_id":"0337f90b-37e7-459a-ab9a-1368af4968bd"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236926318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Activated sub_volume 326baef2-ac5c-47cd-9fab-27ce440cb185","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236934428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.236942527Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.237242697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"task":"crutest"} 01/30 Write block 350 len 20480 data: 14 12 13 13 13 02/30 Flush 03/30 Write block 9 len 40960 data: 12 14 11 11 8 10 7 7 9 10 04/30 Read block 373 len 28672 05/30 Read block 581 len 16384 06/30 Write block 205 len 4096 data: 14 07/30 Write block 171 len 16384 data: 12 8 9 12 08/30 Read block 41 len 36864 09/30 Write block 54 len 4096 data: 10 10/30 Read block 427 len 20480 11/30 Read block 269 len 40960 12/30 Read block 398 len 12288 13/30 Read block 38 len 32768 14/30 Read block 371 len 12288 15/30 Read block 99 len 24576 16/30 Read block 135 len 32768 17/30 Write block 201 len 4096 data: 11 18/30 Write block 511 len 20480 data: 10 10 11 13 16 19/30 Write block 58 len 40960 data: 6 9 10 11 9 11 9 9 7 9 20/30 Flush 21/30 Write block 240 len 36864 data: 12 10 12 11 10 11 11 12 11 22/30 Write block 213 len 8192 data: 18 20 23/30 Write block 476 len 12288 data: 10 11 11 24/30 Write block 302 len 28672 data: 12 13 13 13 12 13 13 25/30 Write block 98 len 12288 data: 10 12 13 26/30 Write block 126 len 36864 data: 16 13 12 12 10 13 13 12 15 27/30 Read block 90 len 16384 28/30 Write block 288 len 32768 data: 14 14 13 13 9 8 9 9 29/30 Write block 64 len 8192 data: 10 10 30/30 Write block 563 len 20480 data: 12 12 12 13 16 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 24 3 30 Done 25 4 25 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1001)) Some(JobId(1019)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:37.245163296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:37.245354584Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1731,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 146 148 265 261 265 F T F <--- 1 020-039 146 146 146 261 261 261 F F F 2 040-059 148 144 148 265 256 265 F F F <--- 3 060-079 148 145 148 265 260 265 T F T <--- 4 080-099 146 146 146 261 261 261 T F T <--- 5 100-119 147 147 147 263 263 263 T F T <--- 6 120-139 147 147 147 263 263 263 T F T <--- 7 140-159 143 143 143 255 255 255 F F F 8 160-179 148 147 148 265 263 265 F F F <--- 9 180-199 147 147 147 263 263 263 F F F 10 200-219 148 147 148 265 263 265 T T T <--- 11 220-239 142 142 142 251 251 251 F F F 12 240-259 143 143 143 255 255 255 T F T <--- 13 260-279 147 147 147 263 263 263 F F F 14 280-299 145 145 145 260 260 260 T F T <--- 15 300-319 146 146 146 262 262 262 T F T <--- 16 320-339 142 142 142 251 251 251 F F F 17 340-359 148 148 148 264 264 264 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 144 144 144 259 259 259 T F T <--- 24 480-499 142 142 142 254 254 254 F F F 25 500-519 148 146 148 265 262 265 F F F <--- 26 520-539 146 146 146 262 262 262 F F F 27 540-559 145 145 145 260 260 260 F F F 28 560-579 145 145 145 260 260 260 T F T <--- 29 580-599 147 147 147 263 263 263 F F F Max gen: 148, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 148 148 148 265 265 265 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 149 149 149 266 266 266 F F F 5 100-119 149 149 149 266 266 266 F F F 6 120-139 149 149 149 266 266 266 F F F 7 140-159 143 143 143 255 255 255 F F F 8 160-179 148 148 148 265 265 265 F F F 9 180-199 147 147 147 263 263 263 F F F 10 200-219 149 149 149 266 266 266 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 149 149 149 266 266 266 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 149 149 149 266 266 266 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 148 148 148 264 264 264 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 140 140 140 248 248 248 F F F 23 460-479 149 149 149 266 266 266 F F F 24 480-499 142 142 142 254 254 254 F F F 25 500-519 148 148 148 265 265 265 F F F 26 520-539 146 146 146 262 262 262 F F F 27 540-559 145 145 145 260 260 260 F F F 28 560-579 149 149 149 266 266 266 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 149, Max flush: 266 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.505040892Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513311438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513600899Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513628647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513637206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513648885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Crucible 4df21930-67cd-4497-83ff-038fdd2f907f has session id: 943e940d-b539-4845-9626-dba4a7183b9e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513928117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Upstairs opts: Upstairs UUID: 4df21930-67cd-4497-83ff-038fdd2f907f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513947536Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Crucible stats registered with UUID: 4df21930-67cd-4497-83ff-038fdd2f907f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.513957605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:41.514001542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514166631Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514197369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514208529Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514372528Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514389187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"4df21930-67cd-4497-83ff-038fdd2f907f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514412825Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514424834Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"0","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514434964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"1","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514444593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"2","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514454022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514463922Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514475481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65135)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514500879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46349)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.51463712Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43728)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.514657599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515172705Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"1","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515198654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"0","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515211123Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"2","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515497614Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515508723Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[0]R flush_numbers[0..12]: [265, 261, 265, 266, 266, 266, 266, 255, 265, 263, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515545191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[0]R generation[0..12]: [148, 146, 148, 149, 149, 149, 149, 143, 148, 147, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.51555888Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515570359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[1]R flush_numbers[0..12]: [265, 261, 265, 266, 266, 266, 266, 255, 265, 263, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515581198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[1]R generation[0..12]: [148, 146, 148, 149, 149, 149, 149, 143, 148, 147, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515591808Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515602967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[2]R flush_numbers[0..12]: [265, 261, 265, 266, 266, 266, 266, 255, 265, 263, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515613666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[2]R generation[0..12]: [148, 146, 148, 149, 149, 149, 149, 143, 148, 147, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515624366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515635485Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515644894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515654684Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515664013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515673342Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"":"downstairs","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515682962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515691801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"0","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.51570154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"1","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.51571257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"client":"2","":"downstairs","session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515722389Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515731578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"4df21930-67cd-4497-83ff-038fdd2f907f is now active with session: 943e940d-b539-4845-9626-dba4a7183b9e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515741618Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515750367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"session_id":"943e940d-b539-4845-9626-dba4a7183b9e"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515759557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Activated sub_volume 4df21930-67cd-4497-83ff-038fdd2f907f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515767516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.515775576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.516070356Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"task":"crutest"} 01/30 Write block 500 len 4096 data: 9 02/30 Read block 109 len 32768 03/30 Read block 340 len 16384 04/30 Read block 429 len 32768 05/30 Flush 06/30 Write block 102 len 4096 data: 14 07/30 Read block 501 len 8192 08/30 Read block 517 len 8192 09/30 Read block 263 len 32768 10/30 Write block 532 len 8192 data: 13 12 11/30 Write block 170 len 8192 data: 10 13 12/30 Write block 140 len 40960 data: 10 10 14 13 13 13 13 17 18 22 13/30 Write block 438 len 4096 data: 13 14/30 Flush 15/30 Write block 511 len 12288 data: 11 11 12 16/30 Write block 546 len 20480 data: 12 15 14 15 13 17/30 Write block 511 len 36864 data: 12 12 13 14 17 18 16 12 11 18/30 Write block 531 len 28672 data: 12 14 13 11 13 15 15 19/30 Read block 267 len 32768 20/30 Write block 245 len 40960 data: 12 12 13 12 10 14 12 11 12 10 21/30 Write block 437 len 40960 data: 11 14 10 9 9 10 10 10 13 13 22/30 Read block 118 len 36864 23/30 Read block 596 len 4096 24/30 Read block 451 len 36864 25/30 Read block 245 len 16384 26/30 Write block 188 len 28672 data: 16 13 13 7 9 9 9 27/30 Read block 293 len 12288 28/30 Write block 310 len 24576 data: 15 13 12 11 11 9 29/30 Write block 560 len 32768 data: 14 14 13 13 13 13 14 17 30/30 Write block 478 len 16384 data: 12 12 11 12 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 29 34 Done 28 27 1 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:41.523932219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:41.524127066Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1739,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 148 148 148 265 265 265 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 149 149 149 266 266 266 F F F 5 100-119 150 150 149 268 268 266 F F F <--- 6 120-139 149 149 149 266 266 266 F F F 7 140-159 150 150 143 268 268 255 F F F <--- 8 160-179 150 150 148 268 268 265 F F F <--- 9 180-199 147 147 147 263 263 263 T T F <--- 10 200-219 149 149 149 266 266 266 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 149 149 149 266 266 266 T T F <--- 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 149 149 149 266 266 266 T T F <--- 16 320-339 142 142 142 251 251 251 F F F 17 340-359 148 148 148 264 264 264 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 150 150 137 268 268 244 T T F <--- 22 440-459 140 140 140 248 248 248 T T F <--- 23 460-479 149 149 149 266 266 266 T T F <--- 24 480-499 142 142 142 254 254 254 T T F <--- 25 500-519 150 150 148 267 267 265 T T T <--- 26 520-539 150 150 146 268 268 262 T T F <--- 27 540-559 145 145 145 260 260 260 T T F <--- 28 560-579 149 149 149 266 266 266 T T F <--- 29 580-599 147 147 147 263 263 263 F F F Max gen: 150, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 148 148 148 265 265 265 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 149 149 149 266 266 266 F F F 5 100-119 150 150 150 268 268 268 F F F 6 120-139 149 149 149 266 266 266 F F F 7 140-159 150 150 150 268 268 268 F F F 8 160-179 150 150 150 268 268 268 F F F 9 180-199 151 151 151 269 269 269 F F F 10 200-219 149 149 149 266 266 266 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 151 151 151 269 269 269 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 151 151 151 269 269 269 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 148 148 148 264 264 264 F F F 18 360-379 144 144 144 258 258 258 F F F 19 380-399 139 139 139 247 247 247 F F F 20 400-419 139 139 139 247 247 247 F F F 21 420-439 151 151 151 269 269 269 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 151 151 151 269 269 269 F F F 25 500-519 151 151 151 269 269 269 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 151 151 151 269 269 269 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 151, Max flush: 269 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.784130704Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.792438038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.792718039Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.792745368Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.792754077Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.792765816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Crucible d4b8f7a9-6501-4372-b955-f633c758beb8 has session id: ce384b40-b5c5-47db-9382-5d22c5cd127c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793128232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Upstairs opts: Upstairs UUID: d4b8f7a9-6501-4372-b955-f633c758beb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793148971Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Crucible stats registered with UUID: d4b8f7a9-6501-4372-b955-f633c758beb8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:45.7931685Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.79332393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793338919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793348968Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793495598Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793519987Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"d4b8f7a9-6501-4372-b955-f633c758beb8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793535196Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793544265Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"0","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793554424Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"1","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793564204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"2","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793583033Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793593132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793605391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40808)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793667817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63308)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793693205Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42718)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.793747532Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794414438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"0","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794434517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"1","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794445846Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"2","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794785253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794808982Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[0]R flush_numbers[0..12]: [265, 261, 265, 266, 266, 268, 266, 268, 268, 269, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794819791Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[0]R generation[0..12]: [148, 146, 148, 149, 149, 150, 149, 150, 150, 151, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.7948309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.79484241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[1]R flush_numbers[0..12]: [265, 261, 265, 266, 266, 268, 266, 268, 268, 269, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794853539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[1]R generation[0..12]: [148, 146, 148, 149, 149, 150, 149, 150, 150, 151, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794864338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794878457Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[2]R flush_numbers[0..12]: [265, 261, 265, 266, 266, 268, 266, 268, 268, 269, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794889297Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[2]R generation[0..12]: [148, 146, 148, 149, 149, 150, 149, 150, 150, 151, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794899906Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794911155Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794920555Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794930284Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794939613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794949023Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"":"downstairs","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794958912Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794967831Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"0","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.794977741Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"1","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.7949876Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"client":"2","":"downstairs","session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795009919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795019188Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"d4b8f7a9-6501-4372-b955-f633c758beb8 is now active with session: ce384b40-b5c5-47db-9382-5d22c5cd127c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795029197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795037877Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"session_id":"ce384b40-b5c5-47db-9382-5d22c5cd127c"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795048676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Activated sub_volume d4b8f7a9-6501-4372-b955-f633c758beb8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795056666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795064955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.795326818Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"task":"crutest"} 01/30 Write block 434 len 28672 data: 8 10 11 12 15 11 10 02/30 Read block 585 len 32768 03/30 Read block 435 len 4096 04/30 Flush 05/30 Write block 495 len 4096 data: 16 06/30 Flush 07/30 Write block 451 len 12288 data: 13 13 14 08/30 Write block 48 len 12288 data: 9 8 9 09/30 Read block 548 len 36864 10/30 Read block 299 len 40960 11/30 Read block 287 len 32768 12/30 Write block 359 len 24576 data: 11 9 10 11 14 16 13/30 Write block 409 len 28672 data: 8 10 11 11 12 11 11 14/30 Read block 252 len 24576 15/30 Write block 373 len 40960 data: 9 10 11 9 9 11 12 10 10 9 16/30 Read block 209 len 12288 17/30 Read block 308 len 20480 18/30 Read block 93 len 40960 19/30 Read block 487 len 40960 20/30 Read block 556 len 12288 21/30 Write block 195 len 12288 data: 8 10 14 22/30 Read block 198 len 20480 23/30 Read block 176 len 28672 24/30 Write block 431 len 4096 data: 8 25/30 Write block 189 len 32768 data: 14 14 8 10 10 10 9 11 26/30 Read block 187 len 28672 27/30 Write block 93 len 40960 data: 10 9 9 9 8 11 13 14 14 15 28/30 Read block 253 len 32768 29/30 Read block 129 len 20480 30/30 Read block 156 len 24576 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:45.80456193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:45.804737699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1747,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 148 148 148 265 265 265 T T F <--- 3 060-079 149 149 149 266 266 266 F F F 4 080-099 149 149 149 266 266 266 T T F <--- 5 100-119 150 150 150 268 268 268 T T F <--- 6 120-139 149 149 149 266 266 266 F F F 7 140-159 150 150 150 268 268 268 F F F 8 160-179 150 150 150 268 268 268 F F F 9 180-199 151 151 151 269 269 269 T T F <--- 10 200-219 149 149 149 266 266 266 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 151 151 151 269 269 269 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 151 151 151 269 269 269 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 148 148 148 264 264 264 T T F <--- 18 360-379 144 144 144 258 258 258 T T F <--- 19 380-399 139 139 139 247 247 247 T T F <--- 20 400-419 139 139 139 247 247 247 T T F <--- 21 420-439 152 152 151 270 270 269 T T F <--- 22 440-459 152 152 151 270 270 269 T T F <--- 23 460-479 151 151 151 269 269 269 F F F 24 480-499 152 152 151 271 271 269 F F F <--- 25 500-519 151 151 151 269 269 269 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 151 151 151 269 269 269 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 152, Max flush: 271 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 153 153 153 272 272 272 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 153 153 153 272 272 272 F F F 5 100-119 153 153 153 272 272 272 F F F 6 120-139 149 149 149 266 266 266 F F F 7 140-159 150 150 150 268 268 268 F F F 8 160-179 150 150 150 268 268 268 F F F 9 180-199 153 153 153 272 272 272 F F F 10 200-219 149 149 149 266 266 266 F F F 11 220-239 142 142 142 251 251 251 F F F 12 240-259 151 151 151 269 269 269 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 151 151 151 269 269 269 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 153 153 153 272 272 272 F F F 18 360-379 153 153 153 272 272 272 F F F 19 380-399 153 153 153 272 272 272 F F F 20 400-419 153 153 153 272 272 272 F F F 21 420-439 153 153 153 272 272 272 F F F 22 440-459 153 153 153 272 272 272 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 151 151 151 269 269 269 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 151 151 151 269 269 269 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 153, Max flush: 272 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.060393192Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.068620841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.068898253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.068924251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.06893293Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.06894484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Crucible 330ee257-1b01-4378-95b9-f7471bc30a28 has session id: db4d8599-629e-4a29-966a-b3b5900c9875","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.06923498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Upstairs opts: Upstairs UUID: 330ee257-1b01-4378-95b9-f7471bc30a28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069252859Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Crucible stats registered with UUID: 330ee257-1b01-4378-95b9-f7471bc30a28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069263369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:50.069277148Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069433637Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069448776Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069458656Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069611206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069622895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"330ee257-1b01-4378-95b9-f7471bc30a28 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069639964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069648783Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"0","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069658483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"1","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069668072Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"2","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069677811Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069687961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.06969795Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"ds_connection connected from Ok(127.0.0.1:62709)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069800743Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"ds_connection connected from Ok(127.0.0.1:49651)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.069953893Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"ds_connection connected from Ok(127.0.0.1:32810)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.0699944Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070421222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"1","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.07045139Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"0","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070579142Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"2","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070720203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070737082Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[0]R flush_numbers[0..12]: [265, 261, 272, 266, 272, 272, 266, 268, 268, 272, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070898591Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[0]R generation[0..12]: [148, 146, 153, 149, 153, 153, 149, 150, 150, 153, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.07090998Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.07092116Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[1]R flush_numbers[0..12]: [265, 261, 272, 266, 272, 272, 266, 268, 268, 272, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070931879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[1]R generation[0..12]: [148, 146, 153, 149, 153, 153, 149, 150, 150, 153, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070942588Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070953577Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[2]R flush_numbers[0..12]: [265, 261, 272, 266, 272, 272, 266, 268, 268, 272, 266, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070964307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[2]R generation[0..12]: [148, 146, 153, 149, 153, 153, 149, 150, 150, 153, 149, 142]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070974746Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070985675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.070995075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071007194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071016473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071025913Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"":"downstairs","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071035682Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071044611Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"0","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071054211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"1","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.07106378Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"client":"2","":"downstairs","session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.07107363Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071082689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"330ee257-1b01-4378-95b9-f7471bc30a28 is now active with session: db4d8599-629e-4a29-966a-b3b5900c9875","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071092728Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071101438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"session_id":"db4d8599-629e-4a29-966a-b3b5900c9875"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071110587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Activated sub_volume 330ee257-1b01-4378-95b9-f7471bc30a28","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071118527Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071126636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.071427566Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"task":"crutest"} 01/30 Write block 415 len 40960 data: 12 8 9 10 10 10 12 15 11 12 02/30 Flush 03/30 Write block 351 len 24576 data: 13 14 14 14 12 10 04/30 Read block 275 len 20480 05/30 Write block 129 len 8192 data: 13 11 06/30 Read block 170 len 20480 07/30 Read block 228 len 20480 08/30 Write block 167 len 4096 data: 10 09/30 Read block 303 len 32768 10/30 Flush 11/30 Write block 220 len 12288 data: 10 9 8 12/30 Read block 358 len 12288 13/30 Read block 528 len 12288 14/30 Write block 215 len 32768 data: 23 21 16 14 11 11 10 9 15/30 Read block 411 len 32768 16/30 Write block 503 len 4096 data: 10 17/30 Read block 84 len 4096 18/30 Write block 560 len 12288 data: 15 15 14 19/30 Flush 20/30 Write block 438 len 8192 data: 16 12 21/30 Read block 592 len 28672 22/30 Read block 399 len 4096 23/30 Read block 433 len 12288 24/30 Read block 439 len 28672 25/30 Write block 186 len 24576 data: 15 17 17 15 15 9 26/30 Read block 292 len 4096 27/30 Read block 81 len 4096 28/30 Read block 590 len 36864 29/30 Write block 168 len 28672 data: 9 10 11 14 9 10 13 30/30 Read block 379 len 8192 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:50.080197499Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:50.080375278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1755,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 153 153 153 272 272 272 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 153 153 153 272 272 272 F F F 5 100-119 153 153 153 272 272 272 F F F 6 120-139 154 154 149 274 274 266 F F F <--- 7 140-159 150 150 150 268 268 268 F F F 8 160-179 154 154 150 274 274 268 T T F <--- 9 180-199 153 153 153 272 272 272 T T F <--- 10 200-219 154 154 149 275 275 266 F F F <--- 11 220-239 154 154 142 275 275 251 F F F <--- 12 240-259 151 151 151 269 269 269 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 151 151 151 269 269 269 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 154 154 153 274 274 272 F F T <--- 18 360-379 153 153 153 272 272 272 F F F 19 380-399 153 153 153 272 272 272 F F F 20 400-419 154 154 154 273 273 273 F F F 21 420-439 154 154 154 273 273 273 T T F <--- 22 440-459 153 153 153 272 272 272 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 154 154 151 275 275 269 F F F <--- 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 154 154 151 275 275 269 F F F <--- 29 580-599 147 147 147 263 263 263 F F F Max gen: 154, Max flush: 275 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F F F 2 040-059 153 153 153 272 272 272 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 153 153 153 272 272 272 F F F 5 100-119 153 153 153 272 272 272 F F F 6 120-139 154 154 154 274 274 274 F F F 7 140-159 150 150 150 268 268 268 F F F 8 160-179 155 155 155 276 276 276 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 154 154 154 275 275 275 F F F 11 220-239 154 154 154 275 275 275 F F F 12 240-259 151 151 151 269 269 269 F F F 13 260-279 147 147 147 263 263 263 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 151 151 151 269 269 269 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 154 154 154 274 274 274 F F F 18 360-379 153 153 153 272 272 272 F F F 19 380-399 153 153 153 272 272 272 F F F 20 400-419 154 154 154 273 273 273 F F F 21 420-439 155 155 155 276 276 276 F F F 22 440-459 153 153 153 272 272 272 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 154 154 154 275 275 275 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 154 154 154 275 275 275 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 155, Max flush: 276 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.32966232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:54.336921552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.336976489Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344036604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344295787Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344322795Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344331105Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344342794Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Crucible 1ecc5954-a297-445e-aff5-398ed0c2274d has session id: 57df46f6-806f-4d52-8e04-33d9cadb1ff6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344629855Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Upstairs opts: Upstairs UUID: 1ecc5954-a297-445e-aff5-398ed0c2274d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344653834Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Crucible stats registered with UUID: 1ecc5954-a297-445e-aff5-398ed0c2274d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344663793Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:54.344672822Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344814773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344846871Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.34485791Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.344994491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.34501789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"1ecc5954-a297-445e-aff5-398ed0c2274d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345061817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345071436Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"0","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345081135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"1","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345090845Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"2","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345100364Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345110204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345119973Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61285)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345204317Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"ds_connection connected from Ok(127.0.0.1:52236)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345229866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61728)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.34530981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.3459198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"2","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345942919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"1","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.345958348Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"0","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346270127Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346315574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[0]R flush_numbers[0..12]: [265, 261, 272, 266, 272, 272, 274, 268, 276, 276, 275, 275]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346356042Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[0]R generation[0..12]: [148, 146, 153, 149, 153, 153, 154, 150, 155, 155, 154, 154]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346370331Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.34638161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[1]R flush_numbers[0..12]: [265, 261, 272, 266, 272, 272, 274, 268, 276, 276, 275, 275]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346392429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[1]R generation[0..12]: [148, 146, 153, 149, 153, 153, 154, 150, 155, 155, 154, 154]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346402909Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346413958Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[2]R flush_numbers[0..12]: [265, 261, 272, 266, 272, 272, 274, 268, 276, 276, 275, 275]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346424647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[2]R generation[0..12]: [148, 146, 153, 149, 153, 153, 154, 150, 155, 155, 154, 154]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346435206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346446466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346455865Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346465594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346475044Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346484393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"":"downstairs","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346494503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346503332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"0","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346514531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"1","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346524431Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"client":"2","":"downstairs","session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.34653448Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346543839Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"1ecc5954-a297-445e-aff5-398ed0c2274d is now active with session: 57df46f6-806f-4d52-8e04-33d9cadb1ff6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346553889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346562668Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"session_id":"57df46f6-806f-4d52-8e04-33d9cadb1ff6"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346571767Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Activated sub_volume 1ecc5954-a297-445e-aff5-398ed0c2274d","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346579757Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346587866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.346897006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} 01/30 Write block 135 len 4096 data: 14 02/30 Flush 03/30 Write block 249 len 28672 data: 11 15 13 12 13 11 8 04/30 Flush 05/30 Write block 413 len 32768 data: 13 12 13 9 10 11 11 11 06/30 Read block 357 len 20480 07/30 Write block 101 len 40960 data: 15 16 14 12 11 9 11 13 12 13 08/30 Write block 448 len 8192 data: 17 17 09/30 Read block 434 len 4096 10/30 Read block 422 len 32768 11/30 Write block 453 len 40960 data: 15 15 13 12 10 10 10 10 7 8 12/30 Read block 86 len 40960 13/30 Read block 529 len 8192 14/30 Read block 481 len 20480 15/30 Flush 16/30 Write block 404 len 16384 data: 10 11 9 8 17/30 Read block 329 len 4096 18/30 Read block 407 len 24576 19/30 Write block 482 len 40960 data: 14 12 12 12 11 11 14 14 13 10 20/30 Write block 165 len 20480 data: 7 11 11 10 11 21/30 Read block 569 len 40960 22/30 Write block 170 len 16384 data: 12 15 10 11 23/30 Read block 331 len 32768 24/30 Write block 419 len 32768 data: 12 12 13 16 12 13 10 13 25/30 Write block 36 len 40960 data: 8 12 12 10 9 9 10 9 9 8 26/30 Read block 174 len 4096 27/30 Write block 230 len 32768 data: 11 10 7 7 7 11 11 12 28/30 Write block 237 len 32768 data: 13 11 13 13 11 13 12 11 29/30 Write block 264 len 16384 data: 10 10 11 8 30/30 Write block 304 len 24576 data: 14 14 13 14 14 12 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 3 3 36 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:54.355822289Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:54.356027506Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1763,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 146 146 146 261 261 261 F T T <--- 2 040-059 153 153 153 272 272 272 F T T <--- 3 060-079 149 149 149 266 266 266 F F F 4 080-099 153 153 153 272 272 272 F F F 5 100-119 153 156 156 272 279 279 F F F <--- 6 120-139 154 156 156 274 277 277 F F F <--- 7 140-159 150 150 150 268 268 268 F F F 8 160-179 155 155 155 276 276 276 F T T <--- 9 180-199 155 155 155 276 276 276 F F F 10 200-219 154 154 154 275 275 275 F F F 11 220-239 154 154 154 275 275 275 F T T <--- 12 240-259 151 156 156 269 278 278 F T T <--- 13 260-279 147 147 147 263 263 263 F T T <--- 14 280-299 149 149 149 266 266 266 F F F 15 300-319 151 151 151 269 269 269 F T T <--- 16 320-339 142 142 142 251 251 251 F F F 17 340-359 154 154 154 274 274 274 F F F 18 360-379 153 153 153 272 272 272 F F F 19 380-399 153 153 153 272 272 272 F F F 20 400-419 154 156 156 273 279 279 F T T <--- 21 420-439 155 156 156 276 279 279 F T T <--- 22 440-459 153 156 156 272 279 279 F F F <--- 23 460-479 151 156 156 269 279 279 F F F <--- 24 480-499 152 152 152 271 271 271 F T T <--- 25 500-519 154 154 154 275 275 275 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 154 154 154 275 275 275 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 156, Max flush: 279 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 F F F 1 020-039 157 157 157 280 280 280 F F F 2 040-059 157 157 157 280 280 280 F F F 3 060-079 149 149 149 266 266 266 F F F 4 080-099 153 153 153 272 272 272 F F F 5 100-119 156 156 156 279 279 279 F F F 6 120-139 156 156 156 277 277 277 F F F 7 140-159 150 150 150 268 268 268 F F F 8 160-179 157 157 157 280 280 280 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 154 154 154 275 275 275 F F F 11 220-239 157 157 157 280 280 280 F F F 12 240-259 157 157 157 280 280 280 F F F 13 260-279 157 157 157 280 280 280 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 157 157 157 280 280 280 F F F 16 320-339 142 142 142 251 251 251 F F F 17 340-359 154 154 154 274 274 274 F F F 18 360-379 153 153 153 272 272 272 F F F 19 380-399 153 153 153 272 272 272 F F F 20 400-419 157 157 157 280 280 280 F F F 21 420-439 157 157 157 280 280 280 F F F 22 440-459 156 156 156 279 279 279 F F F 23 460-479 156 156 156 279 279 279 F F F 24 480-499 157 157 157 280 280 280 F F F 25 500-519 154 154 154 275 275 275 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 151 151 151 269 269 269 F F F 28 560-579 154 154 154 275 275 275 F F F 29 580-599 147 147 147 263 263 263 F F F Max gen: 157, Max flush: 280 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.621954783Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.630386678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.63066764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.630697138Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.630705797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.630720036Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Crucible 96946e0a-c7af-4efa-88de-13a49219aa9c has session id: e78433f4-67e7-4bd4-97e7-c680dbd01a2a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631096122Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Upstairs opts: Upstairs UUID: 96946e0a-c7af-4efa-88de-13a49219aa9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.63111505Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Crucible stats registered with UUID: 96946e0a-c7af-4efa-88de-13a49219aa9c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.63112494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:58.631134009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631290769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631316527Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631327117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631464038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631479097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"96946e0a-c7af-4efa-88de-13a49219aa9c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631514704Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631524164Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"0","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631533893Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"1","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631543762Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"2","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631553462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631565911Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.6315761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35233)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631648815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56358)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631674744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44066)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.631686143Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632402866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"0","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632428604Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"1","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632440403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"2","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632735324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632769572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[0]R flush_numbers[0..12]: [265, 280, 280, 266, 272, 279, 277, 268, 280, 276, 275, 280]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632823538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[0]R generation[0..12]: [148, 157, 157, 149, 153, 156, 156, 150, 157, 155, 154, 157]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632834787Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632845897Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[1]R flush_numbers[0..12]: [265, 280, 280, 266, 272, 279, 277, 268, 280, 276, 275, 280]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632856586Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[1]R generation[0..12]: [148, 157, 157, 149, 153, 156, 156, 150, 157, 155, 154, 157]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632867315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632881374Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[2]R flush_numbers[0..12]: [265, 280, 280, 266, 272, 279, 277, 268, 280, 276, 275, 280]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632892124Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[2]R generation[0..12]: [148, 157, 157, 149, 153, 156, 156, 150, 157, 155, 154, 157]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632902623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632913522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632922992Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632932731Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.63294206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.63295171Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"":"downstairs","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632961419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632970238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"0","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632979908Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"1","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632989747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"client":"2","":"downstairs","session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.632999557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633008706Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"96946e0a-c7af-4efa-88de-13a49219aa9c is now active with session: e78433f4-67e7-4bd4-97e7-c680dbd01a2a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633018705Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633029075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"session_id":"e78433f4-67e7-4bd4-97e7-c680dbd01a2a"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633038204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Activated sub_volume 96946e0a-c7af-4efa-88de-13a49219aa9c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633046243Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633054473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.633379672Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"task":"crutest"} 01/30 Write block 137 len 4096 data: 15 02/30 Read block 233 len 32768 03/30 Read block 505 len 28672 04/30 Write block 556 len 8192 data: 14 13 05/30 Write block 56 len 28672 data: 10 9 7 10 11 12 10 06/30 Write block 415 len 12288 data: 14 10 11 07/30 Read block 165 len 24576 08/30 Read block 86 len 16384 09/30 Read block 186 len 20480 10/30 Write block 213 len 4096 data: 19 11/30 Read block 437 len 8192 12/30 Read block 44 len 36864 13/30 Flush 14/30 Write block 500 len 40960 data: 10 10 8 11 10 10 9 7 7 9 15/30 Write block 585 len 28672 data: 9 11 11 15 12 13 13 16/30 Read block 495 len 36864 17/30 Write block 466 len 32768 data: 7 9 9 10 10 11 13 11 18/30 Write block 388 len 36864 data: 10 9 11 13 15 15 14 10 8 19/30 Write block 320 len 32768 data: 14 14 15 13 11 8 11 11 20/30 Read block 273 len 36864 21/30 Read block 277 len 24576 22/30 Write block 230 len 4096 data: 12 23/30 Write block 382 len 16384 data: 10 10 9 10 24/30 Flush 25/30 Write block 374 len 4096 data: 11 26/30 Write block 381 len 4096 data: 11 27/30 Read block 550 len 36864 28/30 Read block 505 len 16384 29/30 Read block 228 len 4096 30/30 Write block 6 len 32768 data: 16 16 17 13 15 12 12 9 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 2 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 29 29 3 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:33:58.641864313Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:33:58.642049631Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1771,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 265 265 265 T T F <--- 1 020-039 157 157 157 280 280 280 F F F 2 040-059 158 158 157 281 281 280 F F F <--- 3 060-079 158 158 149 281 281 266 F F F <--- 4 080-099 153 153 153 272 272 272 F F F 5 100-119 156 156 156 279 279 279 F F F 6 120-139 158 158 156 281 281 277 F F T <--- 7 140-159 150 150 150 268 268 268 F F F 8 160-179 157 157 157 280 280 280 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 158 158 154 281 281 275 F F F <--- 11 220-239 158 158 157 282 282 280 F F F <--- 12 240-259 157 157 157 280 280 280 F F F 13 260-279 157 157 157 280 280 280 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 157 157 157 280 280 280 F F F 16 320-339 158 158 142 282 282 251 F F F <--- 17 340-359 154 154 154 274 274 274 F F F 18 360-379 153 153 153 272 272 272 T T F <--- 19 380-399 158 158 153 282 282 272 T T F <--- 20 400-419 158 158 157 281 281 280 F F F <--- 21 420-439 157 157 157 280 280 280 F F F 22 440-459 156 156 156 279 279 279 F F F 23 460-479 158 158 156 282 282 279 F F F <--- 24 480-499 157 157 157 280 280 280 F F F 25 500-519 158 158 154 282 282 275 F F F <--- 26 520-539 151 151 151 269 269 269 F F F 27 540-559 158 158 151 281 281 269 F F F <--- 28 560-579 154 154 154 275 275 275 F F F 29 580-599 158 158 147 282 282 263 F F F <--- Max gen: 158, Max flush: 282 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 283 283 283 F F F 1 020-039 157 157 157 280 280 280 F F F 2 040-059 158 158 158 281 281 281 F F F 3 060-079 158 158 158 281 281 281 F F F 4 080-099 153 153 153 272 272 272 F F F 5 100-119 156 156 156 279 279 279 F F F 6 120-139 158 158 158 281 281 281 F F F 7 140-159 150 150 150 268 268 268 F F F 8 160-179 157 157 157 280 280 280 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 158 158 158 281 281 281 F F F 11 220-239 158 158 158 282 282 282 F F F 12 240-259 157 157 157 280 280 280 F F F 13 260-279 157 157 157 280 280 280 F F F 14 280-299 149 149 149 266 266 266 F F F 15 300-319 157 157 157 280 280 280 F F F 16 320-339 158 158 158 282 282 282 F F F 17 340-359 154 154 154 274 274 274 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 159 159 159 283 283 283 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 157 157 157 280 280 280 F F F 22 440-459 156 156 156 279 279 279 F F F 23 460-479 158 158 158 282 282 282 F F F 24 480-499 157 157 157 280 280 280 F F F 25 500-519 158 158 158 282 282 282 F F F 26 520-539 151 151 151 269 269 269 F F F 27 540-559 158 158 158 281 281 281 F F F 28 560-579 154 154 154 275 275 275 F F F 29 580-599 158 158 158 282 282 282 F F F Max gen: 159, Max flush: 283 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.899550263Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.907730375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908032985Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908058353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908067043Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908079032Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Crucible 560dcf8b-47fa-40e2-85b4-0c87f05e1afb has session id: efba000f-e1fc-451a-ba96-aa18131a06e0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908389362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Upstairs opts: Upstairs UUID: 560dcf8b-47fa-40e2-85b4-0c87f05e1afb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.90841887Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Crucible stats registered with UUID: 560dcf8b-47fa-40e2-85b4-0c87f05e1afb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908430209Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:02.908442068Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908579299Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908595228Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908605247Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908792425Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908804984Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"560dcf8b-47fa-40e2-85b4-0c87f05e1afb active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908839382Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908848801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"0","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908858461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"1","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.90886812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"2","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.90887775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908888079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908898008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50283)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908939165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64148)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908953365Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43165)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.908967924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.909652859Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"2","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.909742673Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"0","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.90977828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"1","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.909996506Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910019104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[0]R flush_numbers[0..12]: [283, 280, 281, 281, 272, 279, 281, 268, 280, 276, 281, 282]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910070531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[0]R generation[0..12]: [159, 157, 158, 158, 153, 156, 158, 150, 157, 155, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.91008235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.91009403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[1]R flush_numbers[0..12]: [283, 280, 281, 281, 272, 279, 281, 268, 280, 276, 281, 282]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910104889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[1]R generation[0..12]: [159, 157, 158, 158, 153, 156, 158, 150, 157, 155, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910115528Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910126647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[2]R flush_numbers[0..12]: [283, 280, 281, 281, 272, 279, 281, 268, 280, 276, 281, 282]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910137367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[2]R generation[0..12]: [159, 157, 158, 158, 153, 156, 158, 150, 157, 155, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910148056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910159065Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910171614Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910181574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910191203Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910200592Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"":"downstairs","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910210442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910219261Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"0","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910229041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"1","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.91023875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"client":"2","":"downstairs","session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910248559Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910257779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"560dcf8b-47fa-40e2-85b4-0c87f05e1afb is now active with session: efba000f-e1fc-451a-ba96-aa18131a06e0","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910267828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910276657Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"session_id":"efba000f-e1fc-451a-ba96-aa18131a06e0"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910285697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Activated sub_volume 560dcf8b-47fa-40e2-85b4-0c87f05e1afb","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910293816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910301946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.910588037Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"task":"crutest"} 01/30 Write block 419 len 4096 data: 13 02/30 Read block 446 len 12288 03/30 Write block 86 len 36864 data: 9 11 12 12 13 14 12 11 10 04/30 Read block 83 len 36864 05/30 Read block 406 len 12288 06/30 Write block 564 len 24576 data: 14 14 15 18 15 14 07/30 Write block 319 len 20480 data: 14 15 15 16 14 08/30 Write block 476 len 8192 data: 11 12 09/30 Write block 111 len 32768 data: 14 14 11 11 12 12 12 12 10/30 Write block 294 len 20480 data: 10 10 8 8 9 11/30 Read block 378 len 32768 12/30 Flush 13/30 Write block 143 len 24576 data: 14 14 14 14 18 19 14/30 Flush 15/30 Write block 266 len 32768 data: 12 9 12 12 12 12 11 13 16/30 Write block 26 len 36864 data: 11 9 10 10 9 10 9 8 9 17/30 Write block 569 len 40960 data: 15 12 12 10 8 9 10 12 10 9 18/30 Write block 461 len 8192 data: 8 9 19/30 Read block 123 len 20480 20/30 Read block 502 len 28672 21/30 Read block 565 len 24576 22/30 Read block 394 len 32768 23/30 Flush 24/30 Write block 515 len 36864 data: 18 19 17 13 12 13 8 7 10 25/30 Write block 420 len 24576 data: 13 14 17 13 14 11 26/30 Read block 24 len 36864 27/30 Read block 594 len 16384 28/30 Write block 217 len 36864 data: 17 15 12 12 11 10 8 8 8 29/30 Read block 295 len 4096 30/30 Read block 51 len 24576 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 23 1 24 Done 30 7 29 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:02.919619553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:02.919806751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1779,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 283 283 283 F F F 1 020-039 160 157 160 286 280 286 F F F <--- 2 040-059 158 158 158 281 281 281 F F F 3 060-079 158 158 158 281 281 281 F F F 4 080-099 160 153 160 284 272 284 F T F <--- 5 100-119 160 156 160 284 279 284 F F F <--- 6 120-139 158 158 158 281 281 281 F F F 7 140-159 160 150 160 285 268 285 F F F <--- 8 160-179 157 157 157 280 280 280 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 158 158 158 281 281 281 T F T <--- 11 220-239 158 158 158 282 282 282 T F T <--- 12 240-259 157 157 157 280 280 280 F F F 13 260-279 160 157 160 286 280 286 F F F <--- 14 280-299 160 149 160 284 266 284 F F F <--- 15 300-319 160 157 160 284 280 284 F T F <--- 16 320-339 160 158 160 284 282 284 F T F <--- 17 340-359 154 154 154 274 274 274 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 159 159 159 283 283 283 F F F 20 400-419 160 158 160 284 281 284 F T F <--- 21 420-439 157 157 157 280 280 280 T F T <--- 22 440-459 156 156 156 279 279 279 F F F 23 460-479 160 158 160 286 282 286 F F F <--- 24 480-499 157 157 157 280 280 280 F F F 25 500-519 158 158 158 282 282 282 T F T <--- 26 520-539 151 151 151 269 269 269 T F T <--- 27 540-559 158 158 158 281 281 281 F F F 28 560-579 160 154 160 286 275 286 F T F <--- 29 580-599 158 158 158 282 282 282 F F F Max gen: 160, Max flush: 286 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 283 283 283 F F F 1 020-039 160 160 160 286 286 286 F F F 2 040-059 158 158 158 281 281 281 F F F 3 060-079 158 158 158 281 281 281 F F F 4 080-099 160 160 160 284 284 284 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 158 158 158 281 281 281 F F F 7 140-159 160 160 160 285 285 285 F F F 8 160-179 157 157 157 280 280 280 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 161 161 161 287 287 287 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 157 157 157 280 280 280 F F F 13 260-279 160 160 160 286 286 286 F F F 14 280-299 160 160 160 284 284 284 F F F 15 300-319 160 160 160 284 284 284 F F F 16 320-339 160 160 160 284 284 284 F F F 17 340-359 154 154 154 274 274 274 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 159 159 159 283 283 283 F F F 20 400-419 160 160 160 284 284 284 F F F 21 420-439 161 161 161 287 287 287 F F F 22 440-459 156 156 156 279 279 279 F F F 23 460-479 160 160 160 286 286 286 F F F 24 480-499 157 157 157 280 280 280 F F F 25 500-519 161 161 161 287 287 287 F F F 26 520-539 161 161 161 287 287 287 F F F 27 540-559 158 158 158 281 281 281 F F F 28 560-579 160 160 160 286 286 286 F F F 29 580-599 158 158 158 282 282 282 F F F Max gen: 161, Max flush: 287 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.186385665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.194621324Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.194901815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.194932533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.194941382Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.194953252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Crucible a7f23227-68f0-491b-8d5e-d3ee31b63bb8 has session id: d2b93700-5557-4f35-98b3-58c72f0c02ee","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195269731Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Upstairs opts: Upstairs UUID: a7f23227-68f0-491b-8d5e-d3ee31b63bb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195299179Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Crucible stats registered with UUID: a7f23227-68f0-491b-8d5e-d3ee31b63bb8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195310198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:07.195319338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195443799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195473098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195483957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195610628Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195630177Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"a7f23227-68f0-491b-8d5e-d3ee31b63bb8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195657125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195666435Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"0","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195676024Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"1","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195685664Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"2","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195695523Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195705422Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195715482Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:59836)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195783687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:41277)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195808905Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:54976)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.195820215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196461932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"2","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196486641Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"0","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196589334Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"1","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196789981Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.19679941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[0]R flush_numbers[0..12]: [283, 286, 281, 281, 284, 284, 281, 285, 280, 276, 287, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196916163Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[0]R generation[0..12]: [159, 160, 158, 158, 160, 160, 158, 160, 157, 155, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196927522Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196938621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[1]R flush_numbers[0..12]: [283, 286, 281, 281, 284, 284, 281, 285, 280, 276, 287, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.19694941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[1]R generation[0..12]: [159, 160, 158, 158, 160, 160, 158, 160, 157, 155, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.19696Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196971319Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[2]R flush_numbers[0..12]: [283, 286, 281, 281, 284, 284, 281, 285, 280, 276, 287, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196982098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[2]R generation[0..12]: [159, 160, 158, 158, 160, 160, 158, 160, 157, 155, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.196992848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197003797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197013366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197023186Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197032605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197041924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"":"downstairs","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197051654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197062053Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"0","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197071572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"1","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197081222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"client":"2","":"downstairs","session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197091041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197100241Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"a7f23227-68f0-491b-8d5e-d3ee31b63bb8 is now active with session: d2b93700-5557-4f35-98b3-58c72f0c02ee","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.19711038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197119069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"session_id":"d2b93700-5557-4f35-98b3-58c72f0c02ee"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197128169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Activated sub_volume a7f23227-68f0-491b-8d5e-d3ee31b63bb8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197136178Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2026-09-18T03:34:07.197150037Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.197401651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"task":"crutest"} 01/30 Write block 171 len 12288 data: 16 11 12 02/30 Read block 479 len 28672 03/30 Read block 36 len 12288 04/30 Write block 209 len 12288 data: 16 15 20 05/30 Write block 333 len 36864 data: 13 9 10 11 10 12 11 11 13 06/30 Read block 452 len 40960 07/30 Write block 482 len 40960 data: 15 13 13 13 12 12 15 15 14 11 08/30 Flush 09/30 Write block 520 len 24576 data: 14 9 8 11 9 8 10/30 Write block 555 len 32768 data: 13 15 14 15 15 16 16 15 11/30 Flush 12/30 Flush 13/30 Write block 399 len 36864 data: 9 9 8 10 11 11 12 10 9 14/30 Write block 144 len 36864 data: 15 15 15 19 20 23 20 20 17 15/30 Read block 432 len 16384 16/30 Read block 456 len 16384 17/30 Write block 589 len 40960 data: 13 14 14 12 12 15 12 11 10 10 18/30 Read block 376 len 28672 19/30 Read block 559 len 40960 20/30 Write block 23 len 24576 data: 12 11 10 12 10 11 21/30 Write block 408 len 12288 data: 9 9 11 22/30 Write block 445 len 28672 data: 14 14 15 18 18 14 14 23/30 Flush 24/30 Write block 566 len 12288 data: 16 19 16 25/30 Read block 446 len 36864 26/30 Write block 261 len 4096 data: 10 27/30 Read block 525 len 16384 28/30 Read block 51 len 28672 29/30 Read block 496 len 32768 30/30 Write block 585 len 24576 data: 10 12 12 16 14 15 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:07.206506932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:07.206702779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1787,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 283 283 283 F F F 1 020-039 162 160 162 291 286 291 F F F <--- 2 040-059 158 158 158 281 281 281 F F F 3 060-079 158 158 158 281 281 281 F F F 4 080-099 160 160 160 284 284 284 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 158 158 158 281 281 281 F F F 7 140-159 162 160 162 291 285 291 F F F <--- 8 160-179 162 157 162 288 280 288 F T F <--- 9 180-199 155 155 155 276 276 276 F F F 10 200-219 162 161 162 288 287 288 F F F <--- 11 220-239 161 161 161 287 287 287 F F F 12 240-259 157 157 157 280 280 280 F F F 13 260-279 160 160 160 286 286 286 T F T <--- 14 280-299 160 160 160 284 284 284 F F F 15 300-319 160 160 160 284 284 284 F F F 16 320-339 162 160 162 288 284 288 F F F <--- 17 340-359 162 154 162 288 274 288 F F F <--- 18 360-379 159 159 159 283 283 283 F F F 19 380-399 162 159 162 291 283 291 F F F <--- 20 400-419 162 160 162 291 284 291 F F F <--- 21 420-439 161 161 161 287 287 287 F F F 22 440-459 162 156 162 291 279 291 F F F <--- 23 460-479 160 160 160 286 286 286 F F F 24 480-499 162 157 162 288 280 288 F F F <--- 25 500-519 161 161 161 287 287 287 F F F 26 520-539 162 161 162 289 287 289 F F F <--- 27 540-559 162 158 162 289 281 289 F F F <--- 28 560-579 162 160 162 289 286 289 T F T <--- 29 580-599 162 158 162 291 282 291 T F T <--- Max gen: 162, Max flush: 291 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 283 283 283 F F F 1 020-039 162 162 162 291 291 291 F F F 2 040-059 158 158 158 281 281 281 F F F 3 060-079 158 158 158 281 281 281 F F F 4 080-099 160 160 160 284 284 284 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 158 158 158 281 281 281 F F F 7 140-159 162 162 162 291 291 291 F F F 8 160-179 162 162 162 288 288 288 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 162 162 162 288 288 288 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 157 157 157 280 280 280 F F F 13 260-279 163 163 163 292 292 292 F F F 14 280-299 160 160 160 284 284 284 F F F 15 300-319 160 160 160 284 284 284 F F F 16 320-339 162 162 162 288 288 288 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 162 162 162 291 291 291 F F F 20 400-419 162 162 162 291 291 291 F F F 21 420-439 161 161 161 287 287 287 F F F 22 440-459 162 162 162 291 291 291 F F F 23 460-479 160 160 160 286 286 286 F F F 24 480-499 162 162 162 288 288 288 F F F 25 500-519 161 161 161 287 287 287 F F F 26 520-539 162 162 162 289 289 289 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 163 163 163 292 292 292 F F F 29 580-599 163 163 163 292 292 292 F F F Max gen: 163, Max flush: 292 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.464701798Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.472914238Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473208998Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473235996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473247686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473259495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Crucible cda1c21c-caf1-4de5-9d61-82a8e9812e83 has session id: 361c737b-6c50-4d56-86fe-d09e0b53561b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473630651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Upstairs opts: Upstairs UUID: cda1c21c-caf1-4de5-9d61-82a8e9812e83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473661508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Crucible stats registered with UUID: cda1c21c-caf1-4de5-9d61-82a8e9812e83","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473673048Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:11.473682557Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473827348Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473858016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.473869035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474005206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474041533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"cda1c21c-caf1-4de5-9d61-82a8e9812e83 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474050403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474059332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"0","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474068862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"1","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474078571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"2","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.47408812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.47410099Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474110979Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46562)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474183594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53626)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474203533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60106)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474214292Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474873369Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"0","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474898867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"2","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.474910276Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"1","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475212157Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475249694Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[0]R flush_numbers[0..12]: [283, 291, 281, 281, 284, 284, 281, 291, 288, 276, 288, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475301721Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[0]R generation[0..12]: [159, 162, 158, 158, 160, 160, 158, 162, 162, 155, 162, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.47531303Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475324409Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[1]R flush_numbers[0..12]: [283, 291, 281, 281, 284, 284, 281, 291, 288, 276, 288, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475335308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[1]R generation[0..12]: [159, 162, 158, 158, 160, 160, 158, 162, 162, 155, 162, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475349267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475360357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[2]R flush_numbers[0..12]: [283, 291, 281, 281, 284, 284, 281, 291, 288, 276, 288, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475371116Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[2]R generation[0..12]: [159, 162, 158, 158, 160, 160, 158, 162, 162, 155, 162, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475381905Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475392915Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475402364Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475412143Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475421403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475430862Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"":"downstairs","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475440511Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475449311Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"0","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.47545895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"1","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.47546857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"client":"2","":"downstairs","session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475478339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475487518Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"cda1c21c-caf1-4de5-9d61-82a8e9812e83 is now active with session: 361c737b-6c50-4d56-86fe-d09e0b53561b","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475499428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475508207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"session_id":"361c737b-6c50-4d56-86fe-d09e0b53561b"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475517416Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Activated sub_volume cda1c21c-caf1-4de5-9d61-82a8e9812e83","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475525526Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475533535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.475818027Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"task":"crutest"} 01/30 Write block 22 len 12288 data: 13 13 12 02/30 Write block 248 len 16384 data: 13 12 16 14 03/30 Write block 189 len 8192 data: 16 16 04/30 Read block 94 len 16384 05/30 Write block 5 len 8192 data: 14 17 06/30 Read block 524 len 8192 07/30 Read block 513 len 20480 08/30 Write block 451 len 16384 data: 15 14 16 16 09/30 Read block 194 len 12288 10/30 Read block 280 len 40960 11/30 Write block 299 len 28672 data: 11 12 11 13 14 15 15 12/30 Write block 52 len 8192 data: 9 10 13/30 Write block 542 len 24576 data: 15 14 13 16 13 16 14/30 Read block 383 len 36864 15/30 Read block 397 len 4096 16/30 Flush 17/30 Write block 412 len 24576 data: 12 14 13 15 11 12 18/30 Read block 503 len 16384 19/30 Write block 407 len 24576 data: 10 10 10 12 12 13 20/30 Write block 490 len 16384 data: 15 12 14 15 21/30 Write block 2 len 36864 data: 8 10 11 15 18 17 18 14 16 22/30 Read block 447 len 8192 23/30 Write block 2 len 24576 data: 9 11 12 16 19 18 24/30 Read block 572 len 8192 25/30 Write block 141 len 20480 data: 11 15 15 16 16 26/30 Write block 414 len 36864 data: 14 16 12 13 12 14 14 15 18 27/30 Read block 21 len 12288 28/30 Read block 127 len 20480 29/30 Write block 56 len 24576 data: 11 10 8 11 12 13 30/30 Write block 206 len 28672 data: 15 13 17 17 16 21 22 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 28 4 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:11.484103542Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:11.48427593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1795,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 159 164 293 283 293 T F T <--- 1 020-039 164 162 164 293 291 293 F T F <--- 2 040-059 164 158 164 293 281 293 T F T <--- 3 060-079 158 158 158 281 281 281 T F T <--- 4 080-099 160 160 160 284 284 284 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 158 158 158 281 281 281 F F F 7 140-159 162 162 162 291 291 291 T F T <--- 8 160-179 162 162 162 288 288 288 F F F 9 180-199 164 155 164 293 276 293 F T F <--- 10 200-219 162 162 162 288 288 288 T F T <--- 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 157 164 293 280 293 F T F <--- 13 260-279 163 163 163 292 292 292 F F F 14 280-299 164 160 164 293 284 293 F F F <--- 15 300-319 164 160 164 293 284 293 F F F <--- 16 320-339 162 162 162 288 288 288 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 162 162 162 291 291 291 F F F 20 400-419 162 162 162 291 291 291 T F T <--- 21 420-439 161 161 161 287 287 287 T F T <--- 22 440-459 164 162 164 293 291 293 F F F <--- 23 460-479 160 160 160 286 286 286 F F F 24 480-499 162 162 162 288 288 288 T F T <--- 25 500-519 161 161 161 287 287 287 F F F 26 520-539 162 162 162 289 289 289 F F F 27 540-559 164 162 164 293 289 293 F F F <--- 28 560-579 163 163 163 292 292 292 F F F 29 580-599 163 163 163 292 292 292 F F F Max gen: 164, Max flush: 293 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 164 164 164 293 293 293 F F F 2 040-059 165 165 165 294 294 294 F F F 3 060-079 165 165 165 294 294 294 F F F 4 080-099 160 160 160 284 284 284 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 158 158 158 281 281 281 F F F 7 140-159 165 165 165 294 294 294 F F F 8 160-179 162 162 162 288 288 288 F F F 9 180-199 164 164 164 293 293 293 F F F 10 200-219 165 165 165 294 294 294 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 163 163 163 292 292 292 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 164 164 164 293 293 293 F F F 16 320-339 162 162 162 288 288 288 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 162 162 162 291 291 291 F F F 20 400-419 165 165 165 294 294 294 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 160 160 160 286 286 286 F F F 24 480-499 165 165 165 294 294 294 F F F 25 500-519 161 161 161 287 287 287 F F F 26 520-539 162 162 162 289 289 289 F F F 27 540-559 164 164 164 293 293 293 F F F 28 560-579 163 163 163 292 292 292 F F F 29 580-599 163 163 163 292 292 292 F F F Max gen: 165, Max flush: 294 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.748605414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.756751508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757076747Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757103065Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757111624Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757123454Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Crucible f2eb5660-bf40-4dc9-b8ad-bcb8aeffba71 has session id: 7b6a3330-df53-4146-99aa-9330590f8da3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757408585Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Upstairs opts: Upstairs UUID: f2eb5660-bf40-4dc9-b8ad-bcb8aeffba71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757444632Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Crucible stats registered with UUID: f2eb5660-bf40-4dc9-b8ad-bcb8aeffba71","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757457222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:15.757466391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757617631Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.75763296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757643249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757815518Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757832697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"f2eb5660-bf40-4dc9-b8ad-bcb8aeffba71 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757869435Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757878884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"0","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757888603Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"1","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757898103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"2","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757912772Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.757925461Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.75793558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46910)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758003486Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35721)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758030774Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46642)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758125338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758642264Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"1","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758709959Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"0","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758740497Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"2","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.758992281Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759014909Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[0]R flush_numbers[0..12]: [294, 293, 294, 294, 284, 284, 281, 294, 288, 293, 294, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759025678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[0]R generation[0..12]: [165, 164, 165, 165, 160, 160, 158, 165, 162, 164, 165, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759036868Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759048097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[1]R flush_numbers[0..12]: [294, 293, 294, 294, 284, 284, 281, 294, 288, 293, 294, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759058826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[1]R generation[0..12]: [165, 164, 165, 165, 160, 160, 158, 165, 162, 164, 165, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759069396Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759080415Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[2]R flush_numbers[0..12]: [294, 293, 294, 294, 284, 284, 281, 294, 288, 293, 294, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759091274Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[2]R generation[0..12]: [165, 164, 165, 165, 160, 160, 158, 165, 162, 164, 165, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759101873Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759115643Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759125212Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759135051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759144381Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.75915366Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"":"downstairs","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759163559Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759172449Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"0","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759182448Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"1","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759192487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"client":"2","":"downstairs","session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759202657Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759211956Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"f2eb5660-bf40-4dc9-b8ad-bcb8aeffba71 is now active with session: 7b6a3330-df53-4146-99aa-9330590f8da3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759221996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759230665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"session_id":"7b6a3330-df53-4146-99aa-9330590f8da3"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759239934Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Activated sub_volume f2eb5660-bf40-4dc9-b8ad-bcb8aeffba71","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759248074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759256193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.759539605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"task":"crutest"} 01/30 Write block 199 len 28672 data: 14 12 12 11 14 15 15 02/30 Read block 29 len 36864 03/30 Read block 355 len 8192 04/30 Write block 310 len 4096 data: 16 05/30 Write block 499 len 28672 data: 13 11 11 9 12 11 11 06/30 Read block 66 len 16384 07/30 Write block 302 len 12288 data: 14 15 16 08/30 Read block 231 len 24576 09/30 Read block 203 len 16384 10/30 Write block 199 len 20480 data: 15 13 13 12 15 11/30 Write block 150 len 36864 data: 21 21 18 15 17 12 15 17 17 12/30 Read block 245 len 16384 13/30 Write block 194 len 24576 data: 11 10 12 15 15 16 14/30 Flush 15/30 Write block 495 len 20480 data: 17 13 15 14 14 16/30 Write block 92 len 8192 data: 13 12 17/30 Write block 189 len 4096 data: 17 18/30 Read block 377 len 28672 19/30 Read block 88 len 36864 20/30 Write block 93 len 16384 data: 13 11 10 10 21/30 Read block 307 len 20480 22/30 Read block 235 len 28672 23/30 Write block 40 len 24576 data: 10 10 11 10 10 9 24/30 Write block 387 len 16384 data: 10 11 10 12 25/30 Read block 207 len 12288 26/30 Write block 130 len 16384 data: 12 14 14 13 27/30 Write block 320 len 40960 data: 16 16 17 15 12 9 12 12 12 14 28/30 Read block 111 len 20480 29/30 Write block 535 len 12288 data: 14 16 16 30/30 Read block 52 len 32768 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 24 24 Done 30 30 6 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:15.768202075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:15.768388573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1803,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 164 164 164 293 293 293 F F F 2 040-059 165 165 165 294 294 294 T T F <--- 3 060-079 165 165 165 294 294 294 F F F 4 080-099 160 160 160 284 284 284 T T F <--- 5 100-119 160 160 160 284 284 284 F F F 6 120-139 158 158 158 281 281 281 T T F <--- 7 140-159 166 166 165 295 295 294 F F F <--- 8 160-179 162 162 162 288 288 288 F F F 9 180-199 166 166 164 295 295 293 T T T <--- 10 200-219 166 166 165 295 295 294 F F T <--- 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 163 163 163 292 292 292 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 166 166 164 295 295 293 F F T <--- 16 320-339 162 162 162 288 288 288 T T F <--- 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 162 162 162 291 291 291 T T F <--- 20 400-419 165 165 165 294 294 294 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 160 160 160 286 286 286 F F F 24 480-499 166 166 165 295 295 294 T T T <--- 25 500-519 166 166 161 295 295 287 F F T <--- 26 520-539 162 162 162 289 289 289 T T F <--- 27 540-559 164 164 164 293 293 293 F F F 28 560-579 163 163 163 292 292 292 F F F 29 580-599 163 163 163 292 292 292 F F F Max gen: 166, Max flush: 295 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 164 164 164 293 293 293 F F F 2 040-059 167 167 167 296 296 296 F F F 3 060-079 165 165 165 294 294 294 F F F 4 080-099 167 167 167 296 296 296 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 166 166 166 295 295 295 F F F 8 160-179 162 162 162 288 288 288 F F F 9 180-199 167 167 167 296 296 296 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 163 163 163 292 292 292 F F F 14 280-299 164 164 164 293 293 293 F F F 15 300-319 166 166 166 295 295 295 F F F 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 167 167 167 296 296 296 F F F 20 400-419 165 165 165 294 294 294 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 160 160 160 286 286 286 F F F 24 480-499 167 167 167 296 296 296 F F F 25 500-519 166 166 166 295 295 295 F F F 26 520-539 167 167 167 296 296 296 F F F 27 540-559 164 164 164 293 293 293 F F F 28 560-579 163 163 163 292 292 292 F F F 29 580-599 163 163 163 292 292 292 F F F Max gen: 167, Max flush: 296 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.023432446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.031756938Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032057689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032084087Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032092856Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032104796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Crucible 457156ab-a163-4218-b3f8-1625d2eceb4a has session id: bfb0b5e1-1e7d-4586-ae9e-a09683782134","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032403236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Upstairs opts: Upstairs UUID: 457156ab-a163-4218-b3f8-1625d2eceb4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032431394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Crucible stats registered with UUID: 457156ab-a163-4218-b3f8-1625d2eceb4a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032442193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:20.032451633Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032598513Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032613622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032623761Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032784321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032808769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"457156ab-a163-4218-b3f8-1625d2eceb4a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032820979Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032829918Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"0","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032839607Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"1","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032849287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"2","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032858826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032868715Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032878665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"ds_connection connected from Ok(127.0.0.1:34219)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.032983598Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"ds_connection connected from Ok(127.0.0.1:60725)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.033008606Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"ds_connection connected from Ok(127.0.0.1:57958)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.033019965Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.033718219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"2","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.033743158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"0","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.033754947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"1","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034037179Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034049648Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[0]R flush_numbers[0..12]: [294, 293, 296, 294, 296, 284, 296, 295, 288, 296, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034060617Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[0]R generation[0..12]: [165, 164, 167, 165, 167, 160, 167, 166, 162, 167, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034071406Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034082536Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[1]R flush_numbers[0..12]: [294, 293, 296, 294, 296, 284, 296, 295, 288, 296, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034093285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[1]R generation[0..12]: [165, 164, 167, 165, 167, 160, 167, 166, 162, 167, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034103874Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034114933Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[2]R flush_numbers[0..12]: [294, 293, 296, 294, 296, 284, 296, 295, 288, 296, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034125593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[2]R generation[0..12]: [165, 164, 167, 165, 167, 160, 167, 166, 162, 167, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034136282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034147411Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034156751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.03416665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034176339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034185719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"":"downstairs","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034197178Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034205967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"0","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034215707Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"1","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034225226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"client":"2","":"downstairs","session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034235145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034244375Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"457156ab-a163-4218-b3f8-1625d2eceb4a is now active with session: bfb0b5e1-1e7d-4586-ae9e-a09683782134","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034254494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034263354Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"session_id":"bfb0b5e1-1e7d-4586-ae9e-a09683782134"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034272503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Activated sub_volume 457156ab-a163-4218-b3f8-1625d2eceb4a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034280453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.034288622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.03462135Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"task":"crutest"} 01/30 Write block 33 len 40960 data: 9 10 8 9 13 13 11 11 11 12 02/30 Read block 1 len 20480 03/30 Read block 482 len 16384 04/30 Write block 470 len 16384 data: 11 12 14 12 05/30 Write block 178 len 36864 data: 12 13 14 13 13 11 13 14 16 06/30 Write block 283 len 8192 data: 10 10 07/30 Write block 43 len 28672 data: 11 11 10 8 10 10 9 08/30 Write block 149 len 32768 data: 24 22 22 19 16 18 13 16 09/30 Write block 153 len 16384 data: 17 19 14 17 10/30 Read block 569 len 16384 11/30 Flush 12/30 Write block 28 len 20480 data: 12 11 10 11 10 13/30 Flush 14/30 Flush 15/30 Write block 156 len 24576 data: 18 18 18 17 14 14 16/30 Write block 92 len 12288 data: 14 14 12 17/30 Read block 125 len 4096 18/30 Read block 516 len 24576 19/30 Read block 105 len 12288 20/30 Read block 369 len 36864 21/30 Write block 278 len 32768 data: 4 7 6 9 10 11 11 12 22/30 Write block 578 len 28672 data: 10 6 7 6 6 7 7 23/30 Write block 544 len 24576 data: 14 17 14 17 15 16 24/30 Read block 516 len 28672 25/30 Write block 24 len 28672 data: 13 11 13 11 13 12 11 26/30 Flush 27/30 Write block 49 len 4096 data: 10 28/30 Read block 227 len 24576 29/30 Read block 381 len 28672 30/30 Read block 188 len 40960 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:20.043363255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:20.043533484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1811,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 168 164 168 300 293 300 F F F <--- 2 040-059 168 167 168 297 296 297 T F T <--- 3 060-079 165 165 165 294 294 294 F F F 4 080-099 168 167 168 300 296 300 F F F <--- 5 100-119 160 160 160 284 284 284 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 168 166 168 300 295 300 F F F <--- 8 160-179 168 162 168 300 288 300 F F F <--- 9 180-199 168 167 168 297 296 297 F F F <--- 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 168 163 168 300 292 300 F F F <--- 14 280-299 168 164 168 300 293 300 F F F <--- 15 300-319 166 166 166 295 295 295 F F F 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 167 167 167 296 296 296 F F F 20 400-419 165 165 165 294 294 294 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 168 160 168 297 286 297 F F F <--- 24 480-499 167 167 167 296 296 296 F F F 25 500-519 166 166 166 295 295 295 F F F 26 520-539 167 167 167 296 296 296 F F F 27 540-559 168 164 168 300 293 300 F F F <--- 28 560-579 168 163 168 300 292 300 F F F <--- 29 580-599 168 163 168 300 292 300 F F F <--- Max gen: 168, Max flush: 300 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 168 168 168 300 300 300 F F F 2 040-059 169 169 169 301 301 301 F F F 3 060-079 165 165 165 294 294 294 F F F 4 080-099 168 168 168 300 300 300 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 168 168 168 300 300 300 F F F 8 160-179 168 168 168 300 300 300 F F F 9 180-199 168 168 168 297 297 297 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 168 168 168 300 300 300 F F F 14 280-299 168 168 168 300 300 300 F F F 15 300-319 166 166 166 295 295 295 F F F 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 159 159 159 283 283 283 F F F 19 380-399 167 167 167 296 296 296 F F F 20 400-419 165 165 165 294 294 294 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 168 168 168 297 297 297 F F F 24 480-499 167 167 167 296 296 296 F F F 25 500-519 166 166 166 295 295 295 F F F 26 520-539 167 167 167 296 296 296 F F F 27 540-559 168 168 168 300 300 300 F F F 28 560-579 168 168 168 300 300 300 F F F 29 580-599 168 168 168 300 300 300 F F F Max gen: 169, Max flush: 301 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.296843091Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305057701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305346152Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.3053802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305392479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305404578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Crucible 8c1a21d1-6f25-4cb9-9d41-597989b5a19c has session id: 2867466b-a3bd-4bab-83cb-c392dcb104a1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305751835Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Upstairs opts: Upstairs UUID: 8c1a21d1-6f25-4cb9-9d41-597989b5a19c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305794202Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Crucible stats registered with UUID: 8c1a21d1-6f25-4cb9-9d41-597989b5a19c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305805312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:24.305814621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305952332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.305966711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.30597678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.30613278Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306151189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"8c1a21d1-6f25-4cb9-9d41-597989b5a19c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306178637Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306188137Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"0","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306197786Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"1","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306207615Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"2","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306219475Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306242423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306252382Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51190)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306295949Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35600)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306307009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53116)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.306317368Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307010622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"0","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307037211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"2","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.30704927Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"1","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307336401Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307371329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[0]R flush_numbers[0..12]: [294, 300, 301, 294, 300, 284, 296, 300, 300, 297, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307427865Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[0]R generation[0..12]: [165, 168, 169, 165, 168, 160, 167, 168, 168, 168, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307439524Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307450674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[1]R flush_numbers[0..12]: [294, 300, 301, 294, 300, 284, 296, 300, 300, 297, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307461533Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[1]R generation[0..12]: [165, 168, 169, 165, 168, 160, 167, 168, 168, 168, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307474812Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307485931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[2]R flush_numbers[0..12]: [294, 300, 301, 294, 300, 284, 296, 300, 300, 297, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.30749675Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[2]R generation[0..12]: [165, 168, 169, 165, 168, 160, 167, 168, 168, 168, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.3075074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307518599Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307528028Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307537828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307547187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307556527Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"":"downstairs","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307566396Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307575215Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"0","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307585005Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"1","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307594504Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"client":"2","":"downstairs","session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307604323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307613503Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"8c1a21d1-6f25-4cb9-9d41-597989b5a19c is now active with session: 2867466b-a3bd-4bab-83cb-c392dcb104a1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307624902Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307633701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"session_id":"2867466b-a3bd-4bab-83cb-c392dcb104a1"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307642971Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Activated sub_volume 8c1a21d1-6f25-4cb9-9d41-597989b5a19c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.30765095Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.30765902Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.307985078Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"task":"crutest"} 01/30 Write block 178 len 24576 data: 13 14 15 14 14 12 02/30 Read block 526 len 20480 03/30 Write block 24 len 20480 data: 14 12 14 12 14 04/30 Read block 186 len 24576 05/30 Read block 550 len 20480 06/30 Read block 197 len 16384 07/30 Read block 278 len 32768 08/30 Write block 396 len 24576 data: 9 8 8 10 10 9 09/30 Write block 520 len 12288 data: 15 10 9 10/30 Write block 32 len 36864 data: 11 10 11 9 10 14 14 12 12 11/30 Read block 572 len 24576 12/30 Write block 87 len 12288 data: 12 13 13 13/30 Read block 490 len 16384 14/30 Flush 15/30 Write block 293 len 8192 data: 9 11 16/30 Write block 411 len 16384 data: 13 14 15 15 17/30 Write block 262 len 32768 data: 8 8 11 11 13 10 13 13 18/30 Read block 109 len 40960 19/30 Write block 303 len 32768 data: 16 17 16 14 15 15 13 17 20/30 Write block 510 len 8192 data: 12 13 21/30 Write block 578 len 20480 data: 11 7 8 7 7 22/30 Write block 463 len 8192 data: 9 8 23/30 Flush 24/30 Write block 371 len 24576 data: 10 11 10 12 12 10 25/30 Read block 566 len 36864 26/30 Flush 27/30 Write block 556 len 40960 data: 16 15 16 16 17 17 16 14 15 15 28/30 Read block 235 len 36864 29/30 Read block 44 len 4096 30/30 Read block 433 len 28672 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:24.316592402Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:24.31678147Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1819,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 170 170 168 302 302 300 F F F <--- 2 040-059 170 170 169 302 302 301 F F F <--- 3 060-079 165 165 165 294 294 294 F F F 4 080-099 170 170 168 302 302 300 F F F <--- 5 100-119 160 160 160 284 284 284 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 168 168 168 300 300 300 F F F 8 160-179 170 170 168 302 302 300 F F F <--- 9 180-199 170 170 168 302 302 297 F F F <--- 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 170 170 168 303 303 300 F F F <--- 14 280-299 170 170 168 303 303 300 F F F <--- 15 300-319 170 170 166 303 303 295 F F F <--- 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 170 170 159 304 304 283 F F F <--- 19 380-399 170 170 167 302 302 296 F F F <--- 20 400-419 170 170 165 303 303 294 F F F <--- 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 170 170 168 303 303 297 F F F <--- 24 480-499 167 167 167 296 296 296 F F F 25 500-519 170 170 166 303 303 295 F F F <--- 26 520-539 170 170 167 302 302 296 F F F <--- 27 540-559 168 168 168 300 300 300 T T F <--- 28 560-579 170 170 168 303 303 300 T T F <--- 29 580-599 170 170 168 303 303 300 F F F <--- Max gen: 170, Max flush: 304 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 294 294 294 F F F 1 020-039 170 170 170 302 302 302 F F F 2 040-059 170 170 170 302 302 302 F F F 3 060-079 165 165 165 294 294 294 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 168 168 168 300 300 300 F F F 8 160-179 170 170 170 302 302 302 F F F 9 180-199 170 170 170 302 302 302 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 170 170 170 303 303 303 F F F 14 280-299 170 170 170 303 303 303 F F F 15 300-319 170 170 170 303 303 303 F F F 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 170 170 170 304 304 304 F F F 19 380-399 170 170 170 302 302 302 F F F 20 400-419 170 170 170 303 303 303 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 170 170 170 303 303 303 F F F 24 480-499 167 167 167 296 296 296 F F F 25 500-519 170 170 170 303 303 303 F F F 26 520-539 170 170 170 302 302 302 F F F 27 540-559 171 171 171 305 305 305 F F F 28 560-579 171 171 171 305 305 305 F F F 29 580-599 170 170 170 303 303 303 F F F Max gen: 171, Max flush: 305 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.577293904Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.585554771Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.585831303Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.585860881Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.58586932Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.5858813Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Crucible 41ff16fd-c0ae-4315-9723-205f02be79f5 has session id: 4af58c3a-f407-456f-b6c6-7da553df97c8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586217327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Upstairs opts: Upstairs UUID: 41ff16fd-c0ae-4315-9723-205f02be79f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586250995Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Crucible stats registered with UUID: 41ff16fd-c0ae-4315-9723-205f02be79f5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586264654Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:28.586274104Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586424534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586440243Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586450382Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586616631Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.58663358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"41ff16fd-c0ae-4315-9723-205f02be79f5 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586650579Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586659578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"0","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586669388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"1","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586678967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"2","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586688646Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586698636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586723984Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55412)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.58679386Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49588)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586819158Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54394)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.586872234Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587449156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"0","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587469155Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"2","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587482014Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"1","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587788094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587798013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[0]R flush_numbers[0..12]: [294, 302, 302, 294, 302, 284, 296, 300, 302, 302, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587830651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[0]R generation[0..12]: [165, 170, 170, 165, 170, 160, 167, 168, 170, 170, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587842041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.5878532Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[1]R flush_numbers[0..12]: [294, 302, 302, 294, 302, 284, 296, 300, 302, 302, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587864229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[1]R generation[0..12]: [165, 170, 170, 165, 170, 160, 167, 168, 170, 170, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587875118Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587886288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[2]R flush_numbers[0..12]: [294, 302, 302, 294, 302, 284, 296, 300, 302, 302, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587896997Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[2]R generation[0..12]: [165, 170, 170, 165, 170, 160, 167, 168, 170, 170, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587907616Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587921325Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587930925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587940734Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587950193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587959613Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"":"downstairs","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587969282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587978022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"0","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.587987711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"1","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.58800805Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"client":"2","":"downstairs","session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588018239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588027358Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"41ff16fd-c0ae-4315-9723-205f02be79f5 is now active with session: 4af58c3a-f407-456f-b6c6-7da553df97c8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588037348Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588046267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"session_id":"4af58c3a-f407-456f-b6c6-7da553df97c8"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588055357Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Activated sub_volume 41ff16fd-c0ae-4315-9723-205f02be79f5","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588063456Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588071595Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.588357187Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"task":"crutest"} 01/30 Write block 278 len 40960 data: 5 8 7 10 11 12 12 13 12 12 02/30 Read block 570 len 28672 03/30 Write block 158 len 32768 data: 19 18 15 15 11 9 7 8 04/30 Flush 05/30 Write block 10 len 32768 data: 17 13 13 10 11 8 8 10 06/30 Write block 134 len 28672 data: 16 15 13 16 11 11 11 07/30 Read block 112 len 36864 08/30 Write block 525 len 32768 data: 9 8 8 9 10 12 13 15 09/30 Flush 10/30 Write block 371 len 40960 data: 11 12 11 13 13 11 10 12 13 11 11/30 Read block 181 len 24576 12/30 Write block 482 len 28672 data: 16 14 14 14 13 13 16 13/30 Write block 52 len 12288 data: 10 11 11 14/30 Read block 231 len 24576 15/30 Read block 240 len 40960 16/30 Write block 411 len 20480 data: 14 15 16 16 17 17/30 Read block 567 len 8192 18/30 Write block 40 len 20480 data: 13 12 13 12 12 19/30 Read block 466 len 40960 20/30 Write block 165 len 16384 data: 9 12 12 11 21/30 Read block 371 len 24576 22/30 Write block 184 len 8192 data: 14 15 23/30 Read block 82 len 28672 24/30 Flush 25/30 Write block 134 len 16384 data: 17 16 14 17 26/30 Read block 504 len 32768 27/30 Read block 411 len 40960 28/30 Read block 298 len 12288 29/30 Read block 210 len 32768 30/30 Read block 408 len 20480 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 29 1 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:28.597991313Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:28.59818208Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1827,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 165 172 307 294 307 F F F <--- 1 020-039 170 170 170 302 302 302 F F F 2 040-059 172 170 172 308 302 308 F F F <--- 3 060-079 165 165 165 294 294 294 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 172 167 172 307 296 307 T F T <--- 7 140-159 172 168 172 307 300 307 F F F <--- 8 160-179 172 170 172 308 302 308 F F F <--- 9 180-199 172 170 172 308 302 308 F F F <--- 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 172 170 172 306 303 306 F T F <--- 14 280-299 172 170 172 306 303 306 F T F <--- 15 300-319 170 170 170 303 303 303 F F F 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 172 170 172 308 304 308 F F F <--- 19 380-399 172 170 172 308 302 308 F F F <--- 20 400-419 172 170 172 308 303 308 F F F <--- 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 170 170 170 303 303 303 F F F 24 480-499 172 167 172 308 296 308 F F F <--- 25 500-519 170 170 170 303 303 303 F F F 26 520-539 172 170 172 307 302 307 F F F <--- 27 540-559 171 171 171 305 305 305 F F F 28 560-579 171 171 171 305 305 305 F F F 29 580-599 170 170 170 303 303 303 F F F Max gen: 172, Max flush: 308 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 307 307 307 F F F 1 020-039 170 170 170 302 302 302 F F F 2 040-059 172 172 172 308 308 308 F F F 3 060-079 165 165 165 294 294 294 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 173 173 173 309 309 309 F F F 7 140-159 172 172 172 307 307 307 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 172 172 172 308 308 308 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 161 161 161 287 287 287 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 172 172 172 306 306 306 F F F 14 280-299 172 172 172 306 306 306 F F F 15 300-319 170 170 170 303 303 303 F F F 16 320-339 167 167 167 296 296 296 F F F 17 340-359 162 162 162 288 288 288 F F F 18 360-379 172 172 172 308 308 308 F F F 19 380-399 172 172 172 308 308 308 F F F 20 400-419 172 172 172 308 308 308 F F F 21 420-439 165 165 165 294 294 294 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 170 170 170 303 303 303 F F F 24 480-499 172 172 172 308 308 308 F F F 25 500-519 170 170 170 303 303 303 F F F 26 520-539 172 172 172 307 307 307 F F F 27 540-559 171 171 171 305 305 305 F F F 28 560-579 171 171 171 305 305 305 F F F 29 580-599 170 170 170 303 303 303 F F F Max gen: 173, Max flush: 309 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.850454636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.858744751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859023292Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859049271Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.85905788Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859069969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Crucible bfb4b88e-d587-4e05-8cb0-f496aa672ee1 has session id: 2beb6df3-9509-45db-be50-6acda6566d50","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859385429Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Upstairs opts: Upstairs UUID: bfb4b88e-d587-4e05-8cb0-f496aa672ee1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859415017Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Crucible stats registered with UUID: bfb4b88e-d587-4e05-8cb0-f496aa672ee1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859426176Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:32.859435685Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859581296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859596355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859606484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859742875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859759744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"bfb4b88e-d587-4e05-8cb0-f496aa672ee1 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859797372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859806891Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"0","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.85981661Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"1","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.85982647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"2","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859836029Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859846008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859855828Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"ds_connection connected from Ok(127.0.0.1:47022)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859873746Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"ds_connection connected from Ok(127.0.0.1:55251)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.859930853Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"ds_connection connected from Ok(127.0.0.1:41190)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.85996751Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860570191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"1","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.86058353Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"0","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860600919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"2","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860930847Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860942406Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[0]R flush_numbers[0..12]: [307, 302, 308, 294, 302, 284, 309, 307, 308, 308, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860966975Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 165, 170, 160, 173, 172, 172, 172, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860978204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.860989233Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[1]R flush_numbers[0..12]: [307, 302, 308, 294, 302, 284, 309, 307, 308, 308, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861000002Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 165, 170, 160, 173, 172, 172, 172, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861010842Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861021931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[2]R flush_numbers[0..12]: [307, 302, 308, 294, 302, 284, 309, 307, 308, 308, 295, 287]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.86103257Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 165, 170, 160, 173, 172, 172, 172, 166, 161]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.86104317Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861054219Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861063608Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861073318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861082587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861091916Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"":"downstairs","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861103286Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861112075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"0","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861121614Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"1","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861131114Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"client":"2","":"downstairs","session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861140953Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861149943Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"bfb4b88e-d587-4e05-8cb0-f496aa672ee1 is now active with session: 2beb6df3-9509-45db-be50-6acda6566d50","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861159922Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861168651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"session_id":"2beb6df3-9509-45db-be50-6acda6566d50"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861177841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Activated sub_volume bfb4b88e-d587-4e05-8cb0-f496aa672ee1","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.86118582Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.86119403Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.861481301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"task":"crutest"} 01/30 Flush 02/30 Write block 7 len 40960 data: 19 19 15 18 14 14 11 12 9 9 03/30 Read block 79 len 12288 04/30 Read block 266 len 20480 05/30 Write block 336 len 40960 data: 12 11 13 12 12 14 12 12 12 14 06/30 Read block 425 len 32768 07/30 Write block 233 len 16384 data: 8 8 12 12 08/30 Write block 465 len 28672 data: 10 8 10 10 11 12 13 09/30 Write block 353 len 4096 data: 15 10/30 Read block 467 len 24576 11/30 Write block 437 len 12288 data: 13 17 13 12/30 Read block 460 len 8192 13/30 Read block 138 len 8192 14/30 Flush 15/30 Write block 376 len 24576 data: 12 11 13 14 12 12 16/30 Write block 579 len 36864 data: 8 9 8 8 8 8 11 13 13 17/30 Read block 282 len 4096 18/30 Write block 61 len 28672 data: 14 11 12 11 11 8 10 19/30 Read block 432 len 40960 20/30 Read block 314 len 8192 21/30 Write block 220 len 40960 data: 13 12 11 9 9 9 9 12 12 12 22/30 Read block 93 len 16384 23/30 Read block 16 len 20480 24/30 Read block 117 len 4096 25/30 Write block 135 len 36864 data: 17 15 18 12 12 12 12 16 16 26/30 Read block 370 len 28672 27/30 Write block 390 len 8192 data: 13 14 28/30 Read block 171 len 12288 29/30 Write block 182 len 8192 data: 15 13 30/30 Read block 115 len 12288 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:32.870788359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:32.870972896Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1835,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 172 311 311 307 F F F <--- 1 020-039 170 170 170 302 302 302 F F F 2 040-059 172 172 172 308 308 308 F F F 3 060-079 165 165 165 294 294 294 T T F <--- 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 173 173 173 309 309 309 T T F <--- 7 140-159 172 172 172 307 307 307 T T F <--- 8 160-179 172 172 172 308 308 308 F F F 9 180-199 172 172 172 308 308 308 T T F <--- 10 200-219 166 166 166 295 295 295 F F F 11 220-239 174 174 161 311 311 287 T T F <--- 12 240-259 164 164 164 293 293 293 F F F 13 260-279 172 172 172 306 306 306 F F F 14 280-299 172 172 172 306 306 306 F F F 15 300-319 170 170 170 303 303 303 F F F 16 320-339 174 174 167 311 311 296 F F F <--- 17 340-359 174 174 162 311 311 288 F F F <--- 18 360-379 172 172 172 308 308 308 T T F <--- 19 380-399 172 172 172 308 308 308 T T F <--- 20 400-419 172 172 172 308 308 308 F F F 21 420-439 174 174 165 311 311 294 F F F <--- 22 440-459 164 164 164 293 293 293 F F F 23 460-479 174 174 170 311 311 303 F F F <--- 24 480-499 172 172 172 308 308 308 F F F 25 500-519 170 170 170 303 303 303 F F F 26 520-539 172 172 172 307 307 307 F F F 27 540-559 171 171 171 305 305 305 F F F 28 560-579 171 171 171 305 305 305 T T F <--- 29 580-599 170 170 170 303 303 303 T T F <--- Max gen: 174, Max flush: 311 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 311 311 311 F F F 1 020-039 170 170 170 302 302 302 F F F 2 040-059 172 172 172 308 308 308 F F F 3 060-079 175 175 175 312 312 312 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 175 175 175 312 312 312 F F F 7 140-159 175 175 175 312 312 312 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 175 175 175 312 312 312 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 175 175 175 312 312 312 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 172 172 172 306 306 306 F F F 14 280-299 172 172 172 306 306 306 F F F 15 300-319 170 170 170 303 303 303 F F F 16 320-339 174 174 174 311 311 311 F F F 17 340-359 174 174 174 311 311 311 F F F 18 360-379 175 175 175 312 312 312 F F F 19 380-399 175 175 175 312 312 312 F F F 20 400-419 172 172 172 308 308 308 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 174 174 174 311 311 311 F F F 24 480-499 172 172 172 308 308 308 F F F 25 500-519 170 170 170 303 303 303 F F F 26 520-539 172 172 172 307 307 307 F F F 27 540-559 171 171 171 305 305 305 F F F 28 560-579 175 175 175 312 312 312 F F F 29 580-599 175 175 175 312 312 312 F F F Max gen: 175, Max flush: 312 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.131227307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.139456406Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.139747197Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.139781824Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.139793544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.139805543Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Crucible 1218247f-a8ca-48ef-8b4f-0e2736bbba32 has session id: 7b70524f-8676-4f2a-92ad-f303b2240c89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140137321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Upstairs opts: Upstairs UUID: 1218247f-a8ca-48ef-8b4f-0e2736bbba32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140168169Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Crucible stats registered with UUID: 1218247f-a8ca-48ef-8b4f-0e2736bbba32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140179708Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:37.140189488Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140337288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140353267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140363866Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140510077Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140525156Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"1218247f-a8ca-48ef-8b4f-0e2736bbba32 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140567013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140576892Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"0","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140587161Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"1","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140597251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"2","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.14060974Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140620359Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140630799Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:35399)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140679145Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:44450)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140690545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:55720)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.140701244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141395008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"1","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141451935Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"0","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141472623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"2","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141748955Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141783953Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[0]R flush_numbers[0..12]: [311, 302, 308, 312, 302, 284, 312, 312, 308, 312, 295, 312]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141836789Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[0]R generation[0..12]: [174, 170, 172, 175, 170, 160, 175, 175, 172, 175, 166, 175]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141848618Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141859738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[1]R flush_numbers[0..12]: [311, 302, 308, 312, 302, 284, 312, 312, 308, 312, 295, 312]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141870737Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[1]R generation[0..12]: [174, 170, 172, 175, 170, 160, 175, 175, 172, 175, 166, 175]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141884236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141895305Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[2]R flush_numbers[0..12]: [311, 302, 308, 312, 302, 284, 312, 312, 308, 312, 295, 312]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141906025Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[2]R generation[0..12]: [174, 170, 172, 175, 170, 160, 175, 175, 172, 175, 166, 175]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141916534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141927473Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141936833Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141946602Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141956011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141965251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"":"downstairs","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.14197494Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.14198391Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"0","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.141993609Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"1","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142014738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"client":"2","":"downstairs","session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142024607Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142033826Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"1218247f-a8ca-48ef-8b4f-0e2736bbba32 is now active with session: 7b70524f-8676-4f2a-92ad-f303b2240c89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142045106Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142053915Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"session_id":"7b70524f-8676-4f2a-92ad-f303b2240c89"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142063164Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Activated sub_volume 1218247f-a8ca-48ef-8b4f-0e2736bbba32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142071184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142079273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.142421571Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"task":"crutest"} 01/30 Write block 334 len 32768 data: 10 11 13 12 14 13 13 15 02/30 Read block 161 len 32768 03/30 Read block 26 len 8192 04/30 Write block 16 len 20480 data: 10 11 11 11 11 05/30 Read block 462 len 16384 06/30 Flush 07/30 Flush 08/30 Write block 58 len 20480 data: 9 12 13 15 12 09/30 Read block 209 len 8192 10/30 Write block 226 len 16384 data: 10 13 13 13 11/30 Read block 83 len 40960 12/30 Flush 13/30 Write block 363 len 36864 data: 15 17 16 20 19 20 16 12 12 14/30 Write block 233 len 20480 data: 9 9 13 13 14 15/30 Write block 558 len 40960 data: 17 17 18 18 17 15 16 16 17 20 16/30 Write block 408 len 36864 data: 11 11 13 15 16 17 17 18 13 17/30 Read block 69 len 16384 18/30 Flush 19/30 Write block 293 len 28672 data: 10 12 11 9 9 10 12 20/30 Write block 270 len 40960 data: 13 13 12 14 14 9 8 5 6 9 21/30 Read block 533 len 8192 22/30 Write block 393 len 32768 data: 16 15 11 10 9 9 11 11 23/30 Read block 410 len 32768 24/30 Read block 139 len 12288 25/30 Read block 227 len 24576 26/30 Read block 394 len 36864 27/30 Write block 197 len 32768 data: 16 16 17 14 14 13 16 16 28/30 Read block 242 len 24576 29/30 Write block 61 len 24576 data: 16 13 13 12 12 9 30/30 Write block 67 len 20480 data: 11 10 13 14 15 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 18 22 Done 16 16 0 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:37.151778835Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:37.151988332Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1843,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 F F F 1 020-039 176 176 176 313 313 313 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 176 176 176 315 315 315 T T F <--- 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 175 175 175 312 312 312 F F F 7 140-159 175 175 175 312 312 312 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 175 175 175 312 312 312 T T F <--- 10 200-219 166 166 166 295 295 295 T T F <--- 11 220-239 176 176 176 316 316 315 F F F <--- 12 240-259 164 164 164 293 293 293 F F F 13 260-279 172 172 172 306 306 306 T T F <--- 14 280-299 172 172 172 306 306 306 T T F <--- 15 300-319 170 170 170 303 303 303 F F F 16 320-339 176 176 176 313 313 313 F F F 17 340-359 176 176 176 313 313 313 F F F 18 360-379 176 176 175 316 316 312 F F F <--- 19 380-399 175 175 175 312 312 312 T T F <--- 20 400-419 176 176 172 316 316 308 T T F <--- 21 420-439 174 174 174 311 311 311 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 174 174 174 311 311 311 F F F 24 480-499 172 172 172 308 308 308 F F F 25 500-519 170 170 170 303 303 303 F F F 26 520-539 172 172 172 307 307 307 F F F 27 540-559 176 176 171 316 316 305 F F F <--- 28 560-579 176 176 175 316 316 312 F F F <--- 29 580-599 175 175 175 312 312 312 F F F Max gen: 176, Max flush: 316 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 F F F 1 020-039 176 176 176 313 313 313 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 177 177 177 317 317 317 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 160 160 160 284 284 284 F F F 6 120-139 175 175 175 312 312 312 F F F 7 140-159 175 175 175 312 312 312 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 177 177 177 317 317 317 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 176 176 176 316 316 316 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 177 177 177 317 317 317 F F F 14 280-299 177 177 177 317 317 317 F F F 15 300-319 170 170 170 303 303 303 F F F 16 320-339 176 176 176 313 313 313 F F F 17 340-359 176 176 176 313 313 313 F F F 18 360-379 176 176 176 316 316 316 F F F 19 380-399 177 177 177 317 317 317 F F F 20 400-419 177 177 177 317 317 317 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 164 164 164 293 293 293 F F F 23 460-479 174 174 174 311 311 311 F F F 24 480-499 172 172 172 308 308 308 F F F 25 500-519 170 170 170 303 303 303 F F F 26 520-539 172 172 172 307 307 307 F F F 27 540-559 176 176 176 316 316 316 F F F 28 560-579 176 176 176 316 316 316 F F F 29 580-599 175 175 175 312 312 312 F F F Max gen: 177, Max flush: 317 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.406337191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.414642534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.414922236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.414948614Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.414957174Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.414968973Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Crucible 5c8462f5-d5d8-404c-a1b0-c71c7b8e91c7 has session id: 982a13ba-10d2-4a47-b344-1ccad01beea6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415303911Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Upstairs opts: Upstairs UUID: 5c8462f5-d5d8-404c-a1b0-c71c7b8e91c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415332659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Crucible stats registered with UUID: 5c8462f5-d5d8-404c-a1b0-c71c7b8e91c7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415343698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:41.415352848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415511517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415532026Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415542365Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415677686Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415701535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"5c8462f5-d5d8-404c-a1b0-c71c7b8e91c7 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415711144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415720253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"0","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415730453Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"1","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415740272Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"2","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415750081Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415760041Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.41577016Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54816)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415805098Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57891)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415816037Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55950)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.415826316Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416490463Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"1","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416518141Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"0","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416617964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"2","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416816421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.41683538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[0]R flush_numbers[0..12]: [313, 313, 315, 317, 302, 284, 312, 312, 308, 317, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416893246Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 177, 170, 160, 175, 175, 172, 177, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416905005Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416916125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[1]R flush_numbers[0..12]: [313, 313, 315, 317, 302, 284, 312, 312, 308, 317, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416926814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 177, 170, 160, 175, 175, 172, 177, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416937553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416948593Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[2]R flush_numbers[0..12]: [313, 313, 315, 317, 302, 284, 312, 312, 308, 317, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416959462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 177, 170, 160, 175, 175, 172, 177, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.416970121Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.41698106Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.41699056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417002879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417012338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417021768Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"":"downstairs","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417031477Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417040267Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"0","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417050106Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"1","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417059735Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"client":"2","":"downstairs","session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417069535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417078724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"5c8462f5-d5d8-404c-a1b0-c71c7b8e91c7 is now active with session: 982a13ba-10d2-4a47-b344-1ccad01beea6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417088853Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417097693Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"session_id":"982a13ba-10d2-4a47-b344-1ccad01beea6"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417106752Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Activated sub_volume 5c8462f5-d5d8-404c-a1b0-c71c7b8e91c7","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417114622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417122731Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.417421851Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"task":"crutest"} 01/30 Write block 473 len 16384 data: 13 10 10 12 02/30 Flush 03/30 Write block 356 len 8192 data: 11 11 04/30 Write block 184 len 16384 data: 15 16 17 18 05/30 Read block 559 len 32768 06/30 Write block 362 len 36864 data: 12 16 18 17 21 20 21 17 13 07/30 Write block 149 len 12288 data: 25 23 23 08/30 Write block 121 len 12288 data: 12 15 16 09/30 Read block 257 len 32768 10/30 Flush 11/30 Write block 594 len 12288 data: 16 13 12 12/30 Write block 117 len 20480 data: 13 13 13 14 13 13/30 Read block 343 len 16384 14/30 Flush 15/30 Write block 324 len 16384 data: 13 10 13 13 16/30 Write block 454 len 20480 data: 17 14 13 11 11 17/30 Write block 450 len 40960 data: 15 16 15 17 18 15 14 12 12 11 18/30 Read block 341 len 16384 19/30 Write block 139 len 4096 data: 13 20/30 Read block 25 len 4096 21/30 Write block 286 len 12288 data: 13 13 15 22/30 Write block 187 len 8192 data: 19 18 23/30 Read block 562 len 36864 24/30 Flush 25/30 Write block 548 len 20480 data: 16 17 14 11 9 26/30 Write block 502 len 32768 data: 10 13 12 12 10 8 8 10 27/30 Read block 206 len 40960 28/30 Read block 158 len 4096 29/30 Read block 205 len 28672 30/30 Read block 428 len 24576 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:41.425336551Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:41.425520909Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1851,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 F F F 1 020-039 176 176 176 313 313 313 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 177 177 177 317 317 317 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 178 178 160 320 320 284 F F F <--- 6 120-139 178 178 175 321 321 312 F F F <--- 7 140-159 178 178 175 319 319 312 F F F <--- 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 178 177 321 321 317 F F F <--- 10 200-219 177 177 177 317 317 317 F F F 11 220-239 176 176 176 316 316 316 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 177 177 177 317 317 317 F F F 14 280-299 178 178 177 321 321 317 F F F <--- 15 300-319 170 170 170 303 303 303 F F F 16 320-339 178 178 176 321 321 313 F F F <--- 17 340-359 178 178 176 319 319 313 F F F <--- 18 360-379 178 178 176 319 319 316 F F F <--- 19 380-399 177 177 177 317 317 317 F F F 20 400-419 177 177 177 317 317 317 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 178 178 164 321 321 293 F F F <--- 23 460-479 178 178 178 318 318 318 F F F 24 480-499 172 172 172 308 308 308 F F F 25 500-519 170 170 170 303 303 303 T T F <--- 26 520-539 172 172 172 307 307 307 F F F 27 540-559 176 176 176 316 316 316 T T F <--- 28 560-579 176 176 176 316 316 316 F F F 29 580-599 178 178 175 320 320 312 F F F <--- Max gen: 178, Max flush: 321 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 F F F 1 020-039 176 176 176 313 313 313 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 177 177 177 317 317 317 F F F 4 080-099 170 170 170 302 302 302 F F F 5 100-119 178 178 178 320 320 320 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 178 178 178 319 319 319 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 176 176 176 316 316 316 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 177 177 177 317 317 317 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 170 170 170 303 303 303 F F F 16 320-339 178 178 178 321 321 321 F F F 17 340-359 178 178 178 319 319 319 F F F 18 360-379 178 178 178 319 319 319 F F F 19 380-399 177 177 177 317 317 317 F F F 20 400-419 177 177 177 317 317 317 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 178 178 178 321 321 321 F F F 23 460-479 178 178 178 318 318 318 F F F 24 480-499 172 172 172 308 308 308 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 172 172 172 307 307 307 F F F 27 540-559 179 179 179 322 322 322 F F F 28 560-579 176 176 176 316 316 316 F F F 29 580-599 178 178 178 320 320 320 F F F Max gen: 179, Max flush: 322 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.677060605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:45.684312558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.684360344Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.691352035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.691613547Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.691643785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.691652265Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.691663924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Crucible ddc1f648-f932-46ca-8985-dcb042164e2c has session id: 42f76ed1-d682-4c98-bd86-04bb53a1952f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692013291Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Upstairs opts: Upstairs UUID: ddc1f648-f932-46ca-8985-dcb042164e2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692044769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Crucible stats registered with UUID: ddc1f648-f932-46ca-8985-dcb042164e2c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692055258Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:45.692064298Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692204488Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692235556Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692246676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692382717Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692402026Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"ddc1f648-f932-46ca-8985-dcb042164e2c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692431954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692441483Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"0","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692451112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"1","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692460802Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"2","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692470421Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.69248045Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.69249051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64793)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692527037Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51893)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692565035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50307)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.692636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693130498Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"1","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693171185Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"2","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693259189Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"0","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693464136Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693474075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[0]R flush_numbers[0..12]: [313, 313, 315, 317, 302, 320, 321, 319, 308, 321, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693533941Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 177, 170, 178, 178, 178, 172, 178, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.69354529Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.69355666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[1]R flush_numbers[0..12]: [313, 313, 315, 317, 302, 320, 321, 319, 308, 321, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693567419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 177, 170, 178, 178, 178, 172, 178, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693577958Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693588927Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[2]R flush_numbers[0..12]: [313, 313, 315, 317, 302, 320, 321, 319, 308, 321, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693599467Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 177, 170, 178, 178, 178, 172, 178, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693609996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693620945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693630515Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693640334Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693649563Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693658923Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"":"downstairs","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693668652Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693679491Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"0","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693689231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"1","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.69369888Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"client":"2","":"downstairs","session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.69370879Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693718069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"ddc1f648-f932-46ca-8985-dcb042164e2c is now active with session: 42f76ed1-d682-4c98-bd86-04bb53a1952f","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693728078Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693736848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"session_id":"42f76ed1-d682-4c98-bd86-04bb53a1952f"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693745957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Activated sub_volume ddc1f648-f932-46ca-8985-dcb042164e2c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693753967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.693762136Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.694053517Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} 01/30 Write block 495 len 20480 data: 18 14 16 15 15 02/30 Write block 123 len 16384 data: 17 17 16 17 03/30 Write block 437 len 32768 data: 14 18 14 11 10 11 11 11 04/30 Write block 549 len 16384 data: 18 15 12 10 05/30 Read block 204 len 32768 06/30 Read block 46 len 8192 07/30 Write block 550 len 20480 data: 16 13 11 11 11 08/30 Write block 368 len 24576 data: 22 18 14 13 13 12 09/30 Read block 238 len 32768 10/30 Read block 481 len 20480 11/30 Write block 527 len 16384 data: 9 10 11 13 12/30 Read block 213 len 8192 13/30 Flush 14/30 Write block 588 len 4096 data: 17 15/30 Write block 176 len 28672 data: 11 13 14 15 16 15 16 16/30 Read block 403 len 4096 17/30 Read block 516 len 40960 18/30 Write block 313 len 8192 data: 12 12 19/30 Write block 448 len 12288 data: 19 19 16 20/30 Read block 300 len 24576 21/30 Write block 400 len 40960 data: 12 10 11 12 12 13 11 11 12 12 22/30 Write block 404 len 8192 data: 13 14 23/30 Read block 261 len 40960 24/30 Read block 575 len 40960 25/30 Write block 394 len 32768 data: 16 12 11 10 10 12 13 11 26/30 Write block 267 len 40960 data: 11 14 14 14 14 13 15 15 10 9 27/30 Write block 551 len 4096 data: 14 28/30 Read block 129 len 4096 29/30 Write block 77 len 24576 data: 11 7 7 7 9 9 30/30 Write block 284 len 20480 data: 13 14 14 14 16 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 2 2 26 Done 8 28 28 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:45.702230969Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:45.702411057Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1859,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 F F F 1 020-039 176 176 176 313 313 313 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 177 177 177 317 317 317 F T T <--- 4 080-099 170 170 170 302 302 302 F T T <--- 5 100-119 178 178 178 320 320 320 F F F 6 120-139 178 180 180 321 323 323 T F F <--- 7 140-159 178 178 178 319 319 319 F F F 8 160-179 172 172 172 308 308 308 F T T <--- 9 180-199 178 178 178 321 321 321 F T T <--- 10 200-219 177 177 177 317 317 317 F F F 11 220-239 176 176 176 316 316 316 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 177 177 177 317 317 317 F T T <--- 14 280-299 178 178 178 321 321 321 F T T <--- 15 300-319 170 170 170 303 303 303 F T T <--- 16 320-339 178 178 178 321 321 321 F F F 17 340-359 178 178 178 319 319 319 F F F 18 360-379 178 180 180 319 323 323 T F F <--- 19 380-399 177 177 177 317 317 317 F T T <--- 20 400-419 177 177 177 317 317 317 F T T <--- 21 420-439 174 180 180 311 323 323 T F F <--- 22 440-459 178 180 180 321 323 323 T T T <--- 23 460-479 178 178 178 318 318 318 F F F 24 480-499 172 180 180 308 323 323 T F F <--- 25 500-519 179 179 179 322 322 322 F F F 26 520-539 172 180 180 307 323 323 F F F <--- 27 540-559 179 180 180 322 323 323 T T T <--- 28 560-579 176 176 176 316 316 316 F F F 29 580-599 178 178 178 320 320 320 F T T <--- Max gen: 180, Max flush: 323 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 F F F 1 020-039 176 176 176 313 313 313 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 181 181 181 324 324 324 F F F 4 080-099 181 181 181 324 324 324 F F F 5 100-119 178 178 178 320 320 320 F F F 6 120-139 180 180 180 323 323 323 F F F 7 140-159 178 178 178 319 319 319 F F F 8 160-179 181 181 181 324 324 324 F F F 9 180-199 181 181 181 324 324 324 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 176 176 176 316 316 316 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 178 178 178 321 321 321 F F F 17 340-359 178 178 178 319 319 319 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 181 181 181 324 324 324 F F F 21 420-439 180 180 180 323 323 323 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 178 178 178 318 318 318 F F F 24 480-499 180 180 180 323 323 323 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 180 180 180 323 323 323 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 176 176 176 316 316 316 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 181, Max flush: 324 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.978288281Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.986868807Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987155088Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987183316Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987194506Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987206255Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Crucible 8650fdcc-803a-43c8-9f11-3c32d756b310 has session id: 9937284c-7fd0-462d-8091-a8e4610a6435","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987554462Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Upstairs opts: Upstairs UUID: 8650fdcc-803a-43c8-9f11-3c32d756b310, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.9875792Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Crucible stats registered with UUID: 8650fdcc-803a-43c8-9f11-3c32d756b310","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.98758964Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:49.987599009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987746719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987775647Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987786697Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987910339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987922608Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"8650fdcc-803a-43c8-9f11-3c32d756b310 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987954006Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987963445Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"0","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987973154Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"1","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987982904Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"2","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.987992623Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.988004992Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.988015132Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"ds_connection connected from Ok(127.0.0.1:61750)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.988064368Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"ds_connection connected from Ok(127.0.0.1:47877)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.988089017Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"ds_connection connected from Ok(127.0.0.1:48393)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.988153963Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.98865106Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"0","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.988764882Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"1","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.98880021Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"2","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989024315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989050834Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[0]R flush_numbers[0..12]: [313, 313, 315, 324, 324, 320, 323, 319, 324, 324, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989127539Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 181, 181, 178, 180, 178, 181, 181, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989139338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989150567Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[1]R flush_numbers[0..12]: [313, 313, 315, 324, 324, 320, 323, 319, 324, 324, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989161466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 181, 181, 178, 180, 178, 181, 181, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989174565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989185635Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[2]R flush_numbers[0..12]: [313, 313, 315, 324, 324, 320, 323, 319, 324, 324, 317, 316]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989196514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 181, 181, 178, 180, 178, 181, 181, 177, 176]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989207273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989218393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989227772Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989237531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989246961Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.9892565Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"":"downstairs","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989266529Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989275399Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"0","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989285368Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"1","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989295277Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"client":"2","":"downstairs","session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989305307Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989314496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"8650fdcc-803a-43c8-9f11-3c32d756b310 is now active with session: 9937284c-7fd0-462d-8091-a8e4610a6435","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989326495Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989335315Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"session_id":"9937284c-7fd0-462d-8091-a8e4610a6435"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989344394Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Activated sub_volume 8650fdcc-803a-43c8-9f11-3c32d756b310","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989352524Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989360843Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.989658924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"task":"crutest"} 01/30 Write block 233 len 16384 data: 10 10 14 14 02/30 Write block 553 len 40960 data: 12 12 14 17 16 18 18 19 19 18 03/30 Read block 39 len 16384 04/30 Flush 05/30 Write block 435 len 4096 data: 11 06/30 Flush 07/30 Write block 142 len 8192 data: 17 17 08/30 Write block 323 len 4096 data: 16 09/30 Read block 284 len 36864 10/30 Write block 469 len 4096 data: 12 11/30 Read block 170 len 16384 12/30 Read block 20 len 28672 13/30 Read block 464 len 20480 14/30 Write block 409 len 36864 data: 13 14 16 17 18 18 19 14 14 15/30 Write block 424 len 8192 data: 15 12 16/30 Flush 17/30 Write block 522 len 36864 data: 10 12 10 10 9 10 11 12 14 18/30 Read block 482 len 20480 19/30 Write block 492 len 40960 data: 15 16 18 19 15 17 16 16 12 12 20/30 Read block 6 len 24576 21/30 Write block 32 len 4096 data: 12 22/30 Write block 553 len 12288 data: 13 13 15 23/30 Read block 475 len 36864 24/30 Read block 371 len 20480 25/30 Read block 382 len 28672 26/30 Read block 207 len 16384 27/30 Write block 18 len 28672 data: 12 12 12 12 14 14 15 28/30 Read block 541 len 40960 29/30 Write block 543 len 16384 data: 15 15 18 15 30/30 Write block 26 len 24576 data: 15 13 15 13 12 12 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 1 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 24 0 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1003)) Some(JobId(1015)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:49.99838674Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:49.998564488Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1867,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 313 313 313 T F T <--- 1 020-039 176 176 176 313 313 313 T F T <--- 2 040-059 176 176 176 315 315 315 F F F 3 060-079 181 181 181 324 324 324 F F F 4 080-099 181 181 181 324 324 324 F F F 5 100-119 178 178 178 320 320 320 F F F 6 120-139 180 180 180 323 323 323 F F F 7 140-159 182 178 182 327 319 327 F F F <--- 8 160-179 181 181 181 324 324 324 F F F 9 180-199 181 181 181 324 324 324 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 182 182 182 325 325 325 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 182 178 182 327 321 327 F F F <--- 17 340-359 178 178 178 319 319 319 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 182 181 182 327 324 327 F F F <--- 21 420-439 182 180 182 327 323 327 F F F <--- 22 440-459 181 181 181 324 324 324 F F F 23 460-479 182 178 182 327 318 327 F F F <--- 24 480-499 180 180 180 323 323 323 T F T <--- 25 500-519 179 179 179 322 322 322 T F T <--- 26 520-539 180 180 180 323 323 323 T F T <--- 27 540-559 182 182 182 325 325 325 T F T <--- 28 560-579 182 182 182 325 325 325 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 182, Max flush: 327 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 F F F 1 020-039 183 183 183 328 328 328 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 181 181 181 324 324 324 F F F 4 080-099 181 181 181 324 324 324 F F F 5 100-119 178 178 178 320 320 320 F F F 6 120-139 180 180 180 323 323 323 F F F 7 140-159 182 182 182 327 327 327 F F F 8 160-179 181 181 181 324 324 324 F F F 9 180-199 181 181 181 324 324 324 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 182 182 182 325 325 325 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 181 181 181 324 324 324 F F F 16 320-339 182 182 182 327 327 327 F F F 17 340-359 178 178 178 319 319 319 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 181 181 181 324 324 324 F F F 20 400-419 182 182 182 327 327 327 F F F 21 420-439 182 182 182 327 327 327 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 182 182 182 327 327 327 F F F 24 480-499 183 183 183 328 328 328 F F F 25 500-519 183 183 183 328 328 328 F F F 26 520-539 183 183 183 328 328 328 F F F 27 540-559 183 183 183 328 328 328 F F F 28 560-579 182 182 182 325 325 325 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 183, Max flush: 328 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.249977852Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258199021Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258506841Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258569957Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258579426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258591355Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Crucible aad5a85a-d188-402c-b690-97d746a2c82e has session id: 270e13de-f53f-4c4f-89de-3e7496978fd6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258936292Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Upstairs opts: Upstairs UUID: aad5a85a-d188-402c-b690-97d746a2c82e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.258962481Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Crucible stats registered with UUID: aad5a85a-d188-402c-b690-97d746a2c82e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.25897247Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:54.258984419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259137029Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259172137Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259183236Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259319797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259338746Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"aad5a85a-d188-402c-b690-97d746a2c82e active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259365194Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259374764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"0","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259384573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"1","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259394392Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"2","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259404052Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259414121Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.25942407Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54629)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259558262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42995)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.25958594Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40137)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.259596199Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260223468Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"2","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260247576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"0","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260288863Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"1","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260558746Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260601883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[0]R flush_numbers[0..12]: [328, 328, 315, 324, 324, 320, 323, 327, 324, 324, 317, 325]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260673658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[0]R generation[0..12]: [183, 183, 176, 181, 181, 178, 180, 182, 181, 181, 177, 182]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260685087Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260696367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[1]R flush_numbers[0..12]: [328, 328, 315, 324, 324, 320, 323, 327, 324, 324, 317, 325]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260707046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[1]R generation[0..12]: [183, 183, 176, 181, 181, 178, 180, 182, 181, 181, 177, 182]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260717745Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260728775Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[2]R flush_numbers[0..12]: [328, 328, 315, 324, 324, 320, 323, 327, 324, 324, 317, 325]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260739514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[2]R generation[0..12]: [183, 183, 176, 181, 181, 178, 180, 182, 181, 181, 177, 182]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260750253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260761262Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260773772Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260783581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.26079295Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.26080248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"":"downstairs","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260812329Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260821218Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"0","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260831388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"1","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260841367Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"client":"2","":"downstairs","session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260851466Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260860746Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"aad5a85a-d188-402c-b690-97d746a2c82e is now active with session: 270e13de-f53f-4c4f-89de-3e7496978fd6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260870695Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260879475Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"session_id":"270e13de-f53f-4c4f-89de-3e7496978fd6"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260888714Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Activated sub_volume aad5a85a-d188-402c-b690-97d746a2c82e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260896714Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.260904813Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.261214113Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"task":"crutest"} 01/30 Write block 461 len 12288 data: 9 10 10 02/30 Flush 03/30 Write block 302 len 24576 data: 15 17 18 17 15 16 04/30 Write block 515 len 36864 data: 19 20 18 14 13 16 11 11 13 05/30 Read block 514 len 24576 06/30 Write block 391 len 20480 data: 15 16 17 17 13 07/30 Write block 494 len 32768 data: 19 20 16 18 17 17 13 13 08/30 Read block 387 len 16384 09/30 Read block 157 len 8192 10/30 Flush 11/30 Flush 12/30 Write block 543 len 12288 data: 16 16 19 13/30 Read block 540 len 20480 14/30 Flush 15/30 Write block 235 len 20480 data: 15 15 15 12 14 16/30 Read block 370 len 20480 17/30 Read block 553 len 32768 18/30 Write block 323 len 16384 data: 17 14 11 14 19/30 Read block 81 len 24576 20/30 Write block 74 len 28672 data: 15 16 15 12 8 8 8 21/30 Write block 86 len 32768 data: 10 13 14 14 14 15 15 15 22/30 Write block 19 len 16384 data: 13 13 13 15 23/30 Read block 213 len 20480 24/30 Write block 405 len 16384 data: 15 12 12 13 25/30 Write block 270 len 20480 data: 15 15 14 16 16 26/30 Read block 171 len 36864 27/30 Read block 445 len 8192 28/30 Write block 6 len 12288 data: 20 20 20 29/30 Read block 59 len 8192 30/30 Write block 335 len 24576 data: 12 14 13 15 14 14 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:54.269650538Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:54.269840125Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1875,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 328 328 328 T F T <--- 1 020-039 183 183 183 328 328 328 T F T <--- 2 040-059 176 176 176 315 315 315 F F F 3 060-079 181 181 181 324 324 324 T F T <--- 4 080-099 181 181 181 324 324 324 T F T <--- 5 100-119 178 178 178 320 320 320 F F F 6 120-139 180 180 180 323 323 323 F F F 7 140-159 182 182 182 327 327 327 F F F 8 160-179 181 181 181 324 324 324 F F F 9 180-199 181 181 181 324 324 324 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 182 182 182 325 325 325 T F T <--- 12 240-259 164 164 164 293 293 293 F F F 13 260-279 181 181 181 324 324 324 T F T <--- 14 280-299 181 181 181 324 324 324 F F F 15 300-319 184 181 184 330 324 330 F F F <--- 16 320-339 182 182 182 327 327 327 T F T <--- 17 340-359 178 178 178 319 319 319 T F T <--- 18 360-379 180 180 180 323 323 323 F F F 19 380-399 184 181 184 330 324 330 F F F <--- 20 400-419 182 182 182 327 327 327 T F T <--- 21 420-439 182 182 182 327 327 327 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 184 182 184 329 327 329 F T F <--- 24 480-499 184 183 184 330 328 330 F F F <--- 25 500-519 184 183 184 330 328 330 F F F <--- 26 520-539 184 183 184 330 328 330 F F F <--- 27 540-559 184 183 184 332 328 332 F F F <--- 28 560-579 182 182 182 325 325 325 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 184, Max flush: 332 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 333 333 333 F F F 1 020-039 185 185 185 333 333 333 F F F 2 040-059 176 176 176 315 315 315 F F F 3 060-079 185 185 185 333 333 333 F F F 4 080-099 185 185 185 333 333 333 F F F 5 100-119 178 178 178 320 320 320 F F F 6 120-139 180 180 180 323 323 323 F F F 7 140-159 182 182 182 327 327 327 F F F 8 160-179 181 181 181 324 324 324 F F F 9 180-199 181 181 181 324 324 324 F F F 10 200-219 177 177 177 317 317 317 F F F 11 220-239 185 185 185 333 333 333 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 185 185 185 333 333 333 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 184 184 184 330 330 330 F F F 16 320-339 185 185 185 333 333 333 F F F 17 340-359 185 185 185 333 333 333 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 184 184 184 330 330 330 F F F 20 400-419 185 185 185 333 333 333 F F F 21 420-439 182 182 182 327 327 327 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 184 184 184 329 329 329 F F F 24 480-499 184 184 184 330 330 330 F F F 25 500-519 184 184 184 330 330 330 F F F 26 520-539 184 184 184 330 330 330 F F F 27 540-559 184 184 184 332 332 332 F F F 28 560-579 182 182 182 325 325 325 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 185, Max flush: 333 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.532388075Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.540590976Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.540874587Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.540900545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.540909245Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.540921174Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Crucible ecd25552-6673-42a7-8759-e83ed7e2f208 has session id: 80ab74ba-ebe1-43d8-baf1-fac5b8ba462a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541256442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Upstairs opts: Upstairs UUID: ecd25552-6673-42a7-8759-e83ed7e2f208, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.54127764Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Crucible stats registered with UUID: ecd25552-6673-42a7-8759-e83ed7e2f208","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541288Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:58.541297519Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.5414424Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541458159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541468388Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541627057Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541643636Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"ecd25552-6673-42a7-8759-e83ed7e2f208 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541669445Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541679074Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"0","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541688883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"1","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541698713Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"2","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541708362Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541718621Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541728651Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38431)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541816645Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38325)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541936217Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37789)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.541947326Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542530338Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"0","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542542097Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"2","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542619442Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"1","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542859076Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542868756Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[0]R flush_numbers[0..12]: [333, 333, 315, 333, 333, 320, 323, 327, 324, 324, 317, 333]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542932952Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[0]R generation[0..12]: [185, 185, 176, 185, 185, 178, 180, 182, 181, 181, 177, 185]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542944301Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.54295534Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[1]R flush_numbers[0..12]: [333, 333, 315, 333, 333, 320, 323, 327, 324, 324, 317, 333]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542966029Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[1]R generation[0..12]: [185, 185, 176, 185, 185, 178, 180, 182, 181, 181, 177, 185]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542976659Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542987628Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[2]R flush_numbers[0..12]: [333, 333, 315, 333, 333, 320, 323, 327, 324, 324, 317, 333]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.542998327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[2]R generation[0..12]: [185, 185, 176, 185, 185, 178, 180, 182, 181, 181, 177, 185]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543009107Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543020336Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543029875Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543039785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543049174Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543058663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"":"downstairs","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543068663Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543077572Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"0","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543089371Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"1","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543099011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"client":"2","":"downstairs","session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.54310909Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543118229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"ecd25552-6673-42a7-8759-e83ed7e2f208 is now active with session: 80ab74ba-ebe1-43d8-baf1-fac5b8ba462a","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543128449Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543137548Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"session_id":"80ab74ba-ebe1-43d8-baf1-fac5b8ba462a"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543146748Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Activated sub_volume ecd25552-6673-42a7-8759-e83ed7e2f208","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543154867Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543163056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.543476816Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"task":"crutest"} 01/30 Write block 168 len 12288 data: 12 12 13 02/30 Read block 262 len 4096 03/30 Read block 55 len 12288 04/30 Read block 556 len 36864 05/30 Write block 295 len 12288 data: 12 10 10 06/30 Read block 105 len 8192 07/30 Write block 211 len 24576 data: 22 23 20 21 24 22 08/30 Read block 533 len 40960 09/30 Write block 56 len 8192 data: 12 11 10/30 Write block 552 len 16384 data: 12 14 14 16 11/30 Read block 243 len 4096 12/30 Read block 174 len 32768 13/30 Read block 34 len 36864 14/30 Read block 24 len 20480 15/30 Read block 516 len 8192 16/30 Write block 228 len 4096 data: 14 17/30 Read block 409 len 40960 18/30 Write block 409 len 16384 data: 14 15 17 18 19/30 Read block 434 len 12288 20/30 Read block 128 len 20480 21/30 Write block 571 len 20480 data: 13 11 9 10 11 22/30 Write block 405 len 8192 data: 16 13 23/30 Write block 94 len 40960 data: 13 11 11 9 12 14 15 16 17 15 24/30 Write block 94 len 12288 data: 14 12 12 25/30 Read block 219 len 28672 26/30 Write block 312 len 16384 data: 13 13 13 10 27/30 Write block 495 len 20480 data: 21 17 19 18 18 28/30 Read block 284 len 24576 29/30 Write block 315 len 28672 data: 11 11 9 11 15 17 17 30/30 Write block 400 len 36864 data: 14 12 12 13 14 17 14 13 14 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 28 4 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:34:58.55101322Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:34:58.551219457Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1883,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 333 333 333 F F F 1 020-039 185 185 185 333 333 333 F F F 2 040-059 176 176 176 315 315 315 T F T <--- 3 060-079 185 185 185 333 333 333 F F F 4 080-099 185 185 185 333 333 333 T F T <--- 5 100-119 178 178 178 320 320 320 T F T <--- 6 120-139 180 180 180 323 323 323 F F F 7 140-159 182 182 182 327 327 327 F F F 8 160-179 181 181 181 324 324 324 T T T 9 180-199 181 181 181 324 324 324 F F F 10 200-219 177 177 177 317 317 317 T F T <--- 11 220-239 185 185 185 333 333 333 T F T <--- 12 240-259 164 164 164 293 293 293 F F F 13 260-279 185 185 185 333 333 333 F F F 14 280-299 181 181 181 324 324 324 T F T <--- 15 300-319 184 184 184 330 330 330 T F T <--- 16 320-339 185 185 185 333 333 333 T F T <--- 17 340-359 185 185 185 333 333 333 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 184 184 184 330 330 330 F F F 20 400-419 185 185 185 333 333 333 T F T <--- 21 420-439 182 182 182 327 327 327 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 184 184 184 329 329 329 F F F 24 480-499 184 184 184 330 330 330 T F T <--- 25 500-519 184 184 184 330 330 330 F F F 26 520-539 184 184 184 330 330 330 F F F 27 540-559 184 184 184 332 332 332 T F T <--- 28 560-579 182 182 182 325 325 325 T F T <--- 29 580-599 181 181 181 324 324 324 F F F Max gen: 185, Max flush: 333 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 333 333 333 F F F 1 020-039 185 185 185 333 333 333 F F F 2 040-059 186 186 186 334 334 334 F F F 3 060-079 185 185 185 333 333 333 F F F 4 080-099 186 186 186 334 334 334 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 180 180 180 323 323 323 F F F 7 140-159 182 182 182 327 327 327 F F F 8 160-179 186 186 186 334 334 334 F F F 9 180-199 181 181 181 324 324 324 F F F 10 200-219 186 186 186 334 334 334 F F F 11 220-239 186 186 186 334 334 334 F F F 12 240-259 164 164 164 293 293 293 F F F 13 260-279 185 185 185 333 333 333 F F F 14 280-299 186 186 186 334 334 334 F F F 15 300-319 186 186 186 334 334 334 F F F 16 320-339 186 186 186 334 334 334 F F F 17 340-359 185 185 185 333 333 333 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 184 184 184 330 330 330 F F F 20 400-419 186 186 186 334 334 334 F F F 21 420-439 182 182 182 327 327 327 F F F 22 440-459 181 181 181 324 324 324 F F F 23 460-479 184 184 184 329 329 329 F F F 24 480-499 186 186 186 334 334 334 F F F 25 500-519 184 184 184 330 330 330 F F F 26 520-539 184 184 184 330 330 330 F F F 27 540-559 186 186 186 334 334 334 F F F 28 560-579 186 186 186 334 334 334 F F F 29 580-599 181 181 181 324 324 324 F F F Max gen: 186, Max flush: 334 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.81387727Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:02.821120184Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.82117624Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.828248865Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.828508597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.828543815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.828556864Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.828569103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Crucible 48ce0a3c-bcab-402b-95f5-0366869e3689 has session id: 8422cc65-3325-4ac9-8d67-8592b25d8d89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.828980296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Upstairs opts: Upstairs UUID: 48ce0a3c-bcab-402b-95f5-0366869e3689, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829000585Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Crucible stats registered with UUID: 48ce0a3c-bcab-402b-95f5-0366869e3689","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829010294Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:02.829019354Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829164564Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829193512Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829204472Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829339683Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829358852Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"48ce0a3c-bcab-402b-95f5-0366869e3689 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.82938719Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829396889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"0","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829406638Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"1","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829416428Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"2","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829427937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829437986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829447936Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"ds_connection connected from Ok(127.0.0.1:40175)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829512701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"ds_connection connected from Ok(127.0.0.1:33542)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.829526091Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"ds_connection connected from Ok(127.0.0.1:39489)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.82953662Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830212445Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"1","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830232004Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"2","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830248323Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"0","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830543724Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830578711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[0]R flush_numbers[0..12]: [333, 333, 334, 333, 334, 334, 323, 327, 334, 324, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830627438Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[0]R generation[0..12]: [185, 185, 186, 185, 186, 186, 180, 182, 186, 181, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830638937Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830650137Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[1]R flush_numbers[0..12]: [333, 333, 334, 333, 334, 334, 323, 327, 334, 324, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830663986Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[1]R generation[0..12]: [185, 185, 186, 185, 186, 186, 180, 182, 186, 181, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830674625Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830685774Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[2]R flush_numbers[0..12]: [333, 333, 334, 333, 334, 334, 323, 327, 334, 324, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830696414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[2]R generation[0..12]: [185, 185, 186, 185, 186, 186, 180, 182, 186, 181, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830707003Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830717952Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Max found gen is 187","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830727531Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Generation requested: 188 >= found:187","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830737291Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.83074672Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.83075646Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"":"downstairs","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830766229Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830774958Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"0","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830784658Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"1","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830794377Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"client":"2","":"downstairs","session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830804346Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830815226Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"48ce0a3c-bcab-402b-95f5-0366869e3689 is now active with session: 8422cc65-3325-4ac9-8d67-8592b25d8d89","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830825385Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830834144Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"session_id":"8422cc65-3325-4ac9-8d67-8592b25d8d89"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830843414Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Activated sub_volume 48ce0a3c-bcab-402b-95f5-0366869e3689","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830851443Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.830859603Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.831144574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} 01/30 Write block 587 len 4096 data: 14 02/30 Read block 164 len 40960 03/30 Read block 283 len 24576 04/30 Read block 395 len 24576 05/30 Read block 54 len 40960 06/30 Write block 178 len 32768 data: 15 16 17 16 17 14 16 17 07/30 Write block 458 len 16384 data: 13 12 11 10 08/30 Read block 393 len 32768 09/30 Write block 343 len 28672 data: 13 13 15 16 17 15 13 10/30 Flush 11/30 Write block 160 len 8192 data: 16 16 12/30 Write block 235 len 28672 data: 16 16 16 13 15 14 12 13/30 Write block 251 len 40960 data: 15 13 14 12 9 9 8 8 7 8 14/30 Write block 131 len 8192 data: 15 15 15/30 Read block 181 len 28672 16/30 Write block 392 len 16384 data: 17 18 18 14 17/30 Read block 276 len 32768 18/30 Write block 202 len 28672 data: 14 17 17 16 16 14 18 19/30 Read block 99 len 20480 20/30 Write block 546 len 16384 data: 16 18 17 19 21/30 Write block 426 len 36864 data: 14 12 9 8 7 9 9 7 9 22/30 Read block 456 len 16384 23/30 Write block 179 len 8192 data: 17 18 24/30 Read block 284 len 24576 25/30 Read block 140 len 36864 26/30 Read block 178 len 4096 27/30 Read block 292 len 36864 28/30 Write block 232 len 20480 data: 8 11 11 17 17 29/30 Write block 213 len 12288 data: 21 22 25 30/30 Read block 558 len 40960 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 3 Sent Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 0 32 Done 0 28 30 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:02.840120484Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:02.840310441Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1891,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 333 333 333 F F F 1 020-039 185 185 185 333 333 333 F F F 2 040-059 186 186 186 334 334 334 F F F 3 060-079 185 185 185 333 333 333 F F F 4 080-099 186 186 186 334 334 334 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 180 180 180 323 323 323 F T T <--- 7 140-159 182 182 182 327 327 327 F F F 8 160-179 186 188 188 334 335 335 F T T <--- 9 180-199 181 188 188 324 335 335 F T T <--- 10 200-219 186 186 186 334 334 334 F T T <--- 11 220-239 186 186 186 334 334 334 F T T <--- 12 240-259 164 164 164 293 293 293 F T T <--- 13 260-279 185 185 185 333 333 333 F T T <--- 14 280-299 186 186 186 334 334 334 F F F 15 300-319 186 186 186 334 334 334 F F F 16 320-339 186 186 186 334 334 334 F F F 17 340-359 185 188 188 333 335 335 F F F <--- 18 360-379 180 180 180 323 323 323 F F F 19 380-399 184 184 184 330 330 330 F T T <--- 20 400-419 186 186 186 334 334 334 F F F 21 420-439 182 182 182 327 327 327 F T T <--- 22 440-459 181 188 188 324 335 335 F F F <--- 23 460-479 184 188 188 329 335 335 F F F <--- 24 480-499 186 186 186 334 334 334 F F F 25 500-519 184 184 184 330 330 330 F F F 26 520-539 184 184 184 330 330 330 F F F 27 540-559 186 186 186 334 334 334 F T T <--- 28 560-579 186 186 186 334 334 334 F F F 29 580-599 181 188 188 324 335 335 F F F <--- Max gen: 188, Max flush: 335 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 333 333 333 F F F 1 020-039 185 185 185 333 333 333 F F F 2 040-059 186 186 186 334 334 334 F F F 3 060-079 185 185 185 333 333 333 F F F 4 080-099 186 186 186 334 334 334 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 189 189 189 336 336 336 F F F 7 140-159 182 182 182 327 327 327 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 F F F 10 200-219 189 189 189 336 336 336 F F F 11 220-239 189 189 189 336 336 336 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 334 334 334 F F F 15 300-319 186 186 186 334 334 334 F F F 16 320-339 186 186 186 334 334 334 F F F 17 340-359 188 188 188 335 335 335 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 189 189 189 336 336 336 F F F 20 400-419 186 186 186 334 334 334 F F F 21 420-439 189 189 189 336 336 336 F F F 22 440-459 188 188 188 335 335 335 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 186 186 186 334 334 334 F F F 25 500-519 184 184 184 330 330 330 F F F 26 520-539 184 184 184 330 330 330 F F F 27 540-559 189 189 189 336 336 336 F F F 28 560-579 186 186 186 334 334 334 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 189, Max flush: 336 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.106313744Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.114934797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115259815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115286514Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115295103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115306732Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Crucible 41cbf11a-7732-44f2-bd59-473d9a0639d8 has session id: e1683bf0-0d74-4eea-a9b1-cedde1c5939e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115620972Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Upstairs opts: Upstairs UUID: 41cbf11a-7732-44f2-bd59-473d9a0639d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115661749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Crucible stats registered with UUID: 41cbf11a-7732-44f2-bd59-473d9a0639d8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115675008Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:07.115684427Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115836817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115866545Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115877755Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.115993987Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116018535Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"41cbf11a-7732-44f2-bd59-473d9a0639d8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116029895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116038884Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"0","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116048583Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"1","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116058193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"2","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116067672Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116077561Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116087551Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46182)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116174785Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41784)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116200143Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58078)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116235971Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.11686628Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"1","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116886948Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"0","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.116898398Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"2","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117185309Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117227796Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[0]R flush_numbers[0..12]: [333, 333, 334, 333, 334, 334, 336, 327, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117280712Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[0]R generation[0..12]: [185, 185, 186, 185, 186, 186, 189, 182, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117291962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117302931Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[1]R flush_numbers[0..12]: [333, 333, 334, 333, 334, 334, 336, 327, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.11731364Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[1]R generation[0..12]: [185, 185, 186, 185, 186, 186, 189, 182, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117324379Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117335479Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[2]R flush_numbers[0..12]: [333, 333, 334, 333, 334, 334, 336, 327, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117346128Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[2]R generation[0..12]: [185, 185, 186, 185, 186, 186, 189, 182, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117359127Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117370206Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117379676Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117389665Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117399165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117408694Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"":"downstairs","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117418553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117427413Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"0","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117437172Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"1","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117447011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"client":"2","":"downstairs","session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117456871Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.11746597Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"41cbf11a-7732-44f2-bd59-473d9a0639d8 is now active with session: e1683bf0-0d74-4eea-a9b1-cedde1c5939e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117476Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117485069Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"session_id":"e1683bf0-0d74-4eea-a9b1-cedde1c5939e"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117494318Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Activated sub_volume 41cbf11a-7732-44f2-bd59-473d9a0639d8","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117502448Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117510617Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.117829426Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"task":"crutest"} 01/30 Write block 389 len 20480 data: 11 14 16 18 19 02/30 Write block 340 len 8192 data: 15 16 03/30 Read block 39 len 40960 04/30 Write block 50 len 24576 data: 10 9 11 12 12 7 05/30 Read block 117 len 8192 06/30 Read block 280 len 16384 07/30 Write block 389 len 24576 data: 12 15 17 19 20 19 08/30 Read block 6 len 20480 09/30 Read block 221 len 4096 10/30 Read block 518 len 24576 11/30 Read block 526 len 4096 12/30 Read block 282 len 16384 13/30 Read block 173 len 16384 14/30 Read block 31 len 4096 15/30 Flush 16/30 Write block 147 len 24576 data: 20 21 26 24 24 20 17/30 Write block 430 len 4096 data: 8 18/30 Read block 512 len 8192 19/30 Write block 35 len 36864 data: 10 11 15 15 13 14 13 14 13 20/30 Read block 218 len 24576 21/30 Write block 0 len 24576 data: 6 8 10 12 13 17 22/30 Write block 43 len 24576 data: 14 13 11 9 11 11 23/30 Read block 310 len 36864 24/30 Write block 511 len 28672 data: 14 13 14 15 20 21 19 25/30 Read block 500 len 16384 26/30 Read block 111 len 36864 27/30 Write block 71 len 12288 data: 16 16 15 28/30 Read block 12 len 32768 29/30 Read block 160 len 40960 30/30 Read block 544 len 40960 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 2 Done Done Sent false 1002 Acked Read 10 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:07.12673871Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:07.126918018Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1899,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 333 333 333 T T F <--- 1 020-039 185 185 185 333 333 333 T T F <--- 2 040-059 190 190 186 337 337 334 T T F <--- 3 060-079 185 185 185 333 333 333 T T F <--- 4 080-099 186 186 186 334 334 334 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 189 189 189 336 336 336 F F F 7 140-159 182 182 182 327 327 327 T T F <--- 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 F F F 10 200-219 189 189 189 336 336 336 F F F 11 220-239 189 189 189 336 336 336 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 334 334 334 F F F 15 300-319 186 186 186 334 334 334 F F F 16 320-339 186 186 186 334 334 334 F F F 17 340-359 190 190 188 337 337 335 F F F <--- 18 360-379 180 180 180 323 323 323 F F F 19 380-399 190 190 189 337 337 336 F F T <--- 20 400-419 186 186 186 334 334 334 F F F 21 420-439 189 189 189 336 336 336 T T F <--- 22 440-459 188 188 188 335 335 335 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 186 186 186 334 334 334 F F F 25 500-519 184 184 184 330 330 330 T T F <--- 26 520-539 184 184 184 330 330 330 F F F 27 540-559 189 189 189 336 336 336 F F F 28 560-579 186 186 186 334 334 334 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 190, Max flush: 337 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 191 191 191 338 338 338 F F F 3 060-079 191 191 191 338 338 338 F F F 4 080-099 186 186 186 334 334 334 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 189 189 189 336 336 336 F F F 7 140-159 191 191 191 338 338 338 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 F F F 10 200-219 189 189 189 336 336 336 F F F 11 220-239 189 189 189 336 336 336 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 334 334 334 F F F 15 300-319 186 186 186 334 334 334 F F F 16 320-339 186 186 186 334 334 334 F F F 17 340-359 190 190 190 337 337 337 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 186 186 186 334 334 334 F F F 21 420-439 191 191 191 338 338 338 F F F 22 440-459 188 188 188 335 335 335 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 186 186 186 334 334 334 F F F 25 500-519 191 191 191 338 338 338 F F F 26 520-539 184 184 184 330 330 330 F F F 27 540-559 189 189 189 336 336 336 F F F 28 560-579 186 186 186 334 334 334 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 191, Max flush: 338 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.374563558Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:11.381850339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.381906925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.388799252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389059615Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389086313Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389094883Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389106622Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Crucible 7e219179-2801-4d0d-8d73-56492306cff6 has session id: a06a43a3-dab0-4d72-8e76-98127d9cda0c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389451889Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Upstairs opts: Upstairs UUID: 7e219179-2801-4d0d-8d73-56492306cff6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389482287Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Crucible stats registered with UUID: 7e219179-2801-4d0d-8d73-56492306cff6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389493446Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:11.389502916Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389632827Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389667235Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389678224Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389796947Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389812056Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"7e219179-2801-4d0d-8d73-56492306cff6 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389828954Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389837854Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"0","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389847433Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"1","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389857253Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"2","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389866842Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389877071Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389887121Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61626)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389947687Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43596)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389973575Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62399)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.389984924Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.390628552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"1","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.3906508Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"2","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.390761393Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"0","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.390950711Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.39096103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[0]R flush_numbers[0..12]: [338, 338, 338, 338, 334, 334, 336, 338, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391068373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[0]R generation[0..12]: [191, 191, 191, 191, 186, 186, 189, 191, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391080022Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391091051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[1]R flush_numbers[0..12]: [338, 338, 338, 338, 334, 334, 336, 338, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391101821Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[1]R generation[0..12]: [191, 191, 191, 191, 186, 186, 189, 191, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.39111252Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391123689Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[2]R flush_numbers[0..12]: [338, 338, 338, 338, 334, 334, 336, 338, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391134409Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[2]R generation[0..12]: [191, 191, 191, 191, 186, 186, 189, 191, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391145058Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391156127Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391165607Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391175516Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391186855Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391196264Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"":"downstairs","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391205894Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391214643Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"0","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391224273Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"1","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391233832Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"client":"2","":"downstairs","session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391243591Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391252701Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"7e219179-2801-4d0d-8d73-56492306cff6 is now active with session: a06a43a3-dab0-4d72-8e76-98127d9cda0c","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.39126279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.3912715Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"session_id":"a06a43a3-dab0-4d72-8e76-98127d9cda0c"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391280779Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Activated sub_volume 7e219179-2801-4d0d-8d73-56492306cff6","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391288868Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391297248Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.391617487Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} 01/30 Write block 56 len 40960 data: 13 12 10 13 14 17 14 14 13 13 02/30 Read block 144 len 36864 03/30 Write block 76 len 24576 data: 16 13 9 9 9 10 04/30 Write block 81 len 32768 data: 11 10 8 8 7 11 14 15 05/30 Write block 436 len 12288 data: 12 15 19 06/30 Read block 321 len 28672 07/30 Read block 560 len 8192 08/30 Write block 73 len 8192 data: 16 16 09/30 Read block 436 len 28672 10/30 Read block 294 len 24576 11/30 Write block 147 len 12288 data: 21 22 27 12/30 Read block 469 len 8192 13/30 Read block 30 len 28672 14/30 Read block 413 len 16384 15/30 Read block 107 len 20480 16/30 Read block 57 len 20480 17/30 Read block 501 len 8192 18/30 Write block 557 len 28672 data: 17 19 19 20 20 19 16 19/30 Write block 556 len 36864 data: 18 18 20 20 21 21 20 17 17 20/30 Flush 21/30 Write block 287 len 24576 data: 15 17 15 14 14 10 22/30 Read block 82 len 12288 23/30 Read block 319 len 24576 24/30 Read block 514 len 4096 25/30 Write block 536 len 24576 data: 17 17 15 13 16 16 26/30 Read block 200 len 32768 27/30 Write block 132 len 32768 data: 16 14 18 18 16 19 13 14 28/30 Write block 305 len 40960 data: 18 16 17 16 14 18 14 14 14 14 29/30 Read block 555 len 32768 30/30 Write block 586 len 32768 data: 14 15 18 15 16 15 13 13 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 11 1 1 13 Done 19 29 29 77 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:11.400719648Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:11.400937024Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1907,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 191 192 192 338 339 339 T F F <--- 3 060-079 191 192 192 338 339 339 T F F <--- 4 080-099 186 192 192 334 339 339 T F F <--- 5 100-119 186 186 186 334 334 334 F F F 6 120-139 189 189 189 336 336 336 F T T <--- 7 140-159 191 192 192 338 339 339 T F F <--- 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 F F F 10 200-219 189 189 189 336 336 336 F F F 11 220-239 189 189 189 336 336 336 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 186 186 186 334 334 334 F T T <--- 15 300-319 186 186 186 334 334 334 F T T <--- 16 320-339 186 186 186 334 334 334 F F F 17 340-359 190 190 190 337 337 337 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 186 186 186 334 334 334 F F F 21 420-439 191 192 192 338 339 339 T F F <--- 22 440-459 188 188 188 335 335 335 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 186 186 186 334 334 334 F F F 25 500-519 191 191 191 338 338 338 F F F 26 520-539 184 184 184 330 330 330 F T T <--- 27 540-559 189 192 192 336 339 339 T T T <--- 28 560-579 186 192 192 334 339 339 T F F <--- 29 580-599 188 188 188 335 335 335 F T T <--- Max gen: 192, Max flush: 339 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 192 192 192 339 339 339 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 193 193 193 340 340 340 F F F 7 140-159 192 192 192 339 339 339 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 F F F 10 200-219 189 189 189 336 336 336 F F F 11 220-239 189 189 189 336 336 336 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 193 193 193 340 340 340 F F F 15 300-319 193 193 193 340 340 340 F F F 16 320-339 186 186 186 334 334 334 F F F 17 340-359 190 190 190 337 337 337 F F F 18 360-379 180 180 180 323 323 323 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 186 186 186 334 334 334 F F F 21 420-439 192 192 192 339 339 339 F F F 22 440-459 188 188 188 335 335 335 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 186 186 186 334 334 334 F F F 25 500-519 191 191 191 338 338 338 F F F 26 520-539 193 193 193 340 340 340 F F F 27 540-559 193 193 193 340 340 340 F F F 28 560-579 192 192 192 339 339 339 F F F 29 580-599 193 193 193 340 340 340 F F F Max gen: 193, Max flush: 340 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.663438797Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.671803286Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672132015Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672161133Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672169502Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672181372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Crucible d677b861-7e1a-4df8-ba6d-9a5f6ee2d263 has session id: 18f2bda1-99a5-4fdc-bd9f-18bbf821743e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672492781Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Upstairs opts: Upstairs UUID: d677b861-7e1a-4df8-ba6d-9a5f6ee2d263, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.67250996Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Crucible stats registered with UUID: d677b861-7e1a-4df8-ba6d-9a5f6ee2d263","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672520239Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:15.672529549Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672678249Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672709117Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672720046Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672857327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672872096Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"d677b861-7e1a-4df8-ba6d-9a5f6ee2d263 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672903634Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672913013Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"0","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672924943Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"1","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672934682Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"2","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672944131Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.672954001Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.6729639Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59767)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.673068993Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55612)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.67311118Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61407)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.67312211Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.673735559Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"0","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.673754758Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"2","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.673787666Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"1","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674065698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674088436Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[0]R flush_numbers[0..12]: [338, 338, 339, 339, 339, 334, 340, 339, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674099245Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[0]R generation[0..12]: [191, 191, 192, 192, 192, 186, 193, 192, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674110635Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674123974Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[1]R flush_numbers[0..12]: [338, 338, 339, 339, 339, 334, 340, 339, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674134733Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[1]R generation[0..12]: [191, 191, 192, 192, 192, 186, 193, 192, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674145222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674156282Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[2]R flush_numbers[0..12]: [338, 338, 339, 339, 339, 334, 340, 339, 336, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674166921Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[2]R generation[0..12]: [191, 191, 192, 192, 192, 186, 193, 192, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.67417757Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.67418885Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674198379Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674208308Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674217578Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674226967Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"":"downstairs","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674236726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674245496Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"0","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674255165Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"1","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674266254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"client":"2","":"downstairs","session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674276094Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674285263Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"d677b861-7e1a-4df8-ba6d-9a5f6ee2d263 is now active with session: 18f2bda1-99a5-4fdc-bd9f-18bbf821743e","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674295333Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674304112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"session_id":"18f2bda1-99a5-4fdc-bd9f-18bbf821743e"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674313201Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Activated sub_volume d677b861-7e1a-4df8-ba6d-9a5f6ee2d263","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674321191Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.67432919Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.674628231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"task":"crutest"} 01/30 Write block 448 len 24576 data: 20 20 17 17 16 18 02/30 Read block 299 len 20480 03/30 Flush 04/30 Write block 549 len 40960 data: 20 17 15 13 15 15 17 19 19 21 05/30 Read block 13 len 12288 06/30 Read block 367 len 36864 07/30 Read block 158 len 16384 08/30 Read block 503 len 28672 09/30 Write block 327 len 24576 data: 14 13 15 15 13 14 10/30 Flush 11/30 Write block 193 len 36864 data: 11 12 11 13 17 17 18 15 15 12/30 Flush 13/30 Write block 120 len 28672 data: 15 14 16 18 18 17 18 14/30 Read block 153 len 12288 15/30 Write block 257 len 24576 data: 9 9 8 9 11 9 16/30 Write block 419 len 20480 data: 15 15 16 19 14 17/30 Read block 76 len 12288 18/30 Read block 146 len 36864 19/30 Flush 20/30 Flush 21/30 Write block 509 len 36864 data: 11 13 15 14 15 16 21 22 20 22/30 Flush 23/30 Write block 192 len 16384 data: 11 12 13 12 24/30 Flush 25/30 Write block 360 len 24576 data: 10 11 13 17 19 18 26/30 Read block 159 len 20480 27/30 Write block 550 len 32768 data: 18 16 14 16 16 18 20 20 28/30 Write block 333 len 4096 data: 14 29/30 Write block 228 len 24576 data: 15 14 13 11 9 12 30/30 Write block 490 len 24576 data: 16 13 16 17 20 22 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 23 4 31 Done 23 4 23 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1002)) Some(JobId(1023)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:15.684356151Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:15.684536619Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1915,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 192 192 192 339 339 339 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 194 193 194 344 340 344 F F F <--- 7 140-159 192 192 192 339 339 339 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 194 189 194 347 336 347 F F F <--- 10 200-219 194 189 194 343 336 343 F F F <--- 11 220-239 189 189 189 336 336 336 T F T <--- 12 240-259 194 189 194 344 336 344 F F F <--- 13 260-279 194 189 194 344 336 344 F F F <--- 14 280-299 193 193 193 340 340 340 F F F 15 300-319 193 193 193 340 340 340 F F F 16 320-339 194 186 194 342 334 342 T F T <--- 17 340-359 190 190 190 337 337 337 F F F 18 360-379 180 180 180 323 323 323 T F T <--- 19 380-399 190 190 190 337 337 337 F F F 20 400-419 194 186 194 344 334 344 F F F <--- 21 420-439 194 192 194 344 339 344 F F F <--- 22 440-459 194 194 194 341 341 341 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 186 186 186 334 334 334 T F T <--- 25 500-519 194 191 194 346 338 346 F F F <--- 26 520-539 193 193 193 340 340 340 F F F 27 540-559 194 193 194 342 340 342 T T T <--- 28 560-579 192 192 192 339 339 339 F F F 29 580-599 193 193 193 340 340 340 F F F Max gen: 194, Max flush: 347 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 192 192 192 339 339 339 F F F 5 100-119 186 186 186 334 334 334 F F F 6 120-139 194 194 194 344 344 344 F F F 7 140-159 192 192 192 339 339 339 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 194 194 194 347 347 347 F F F 10 200-219 194 194 194 343 343 343 F F F 11 220-239 195 195 195 348 348 348 F F F 12 240-259 194 194 194 344 344 344 F F F 13 260-279 194 194 194 344 344 344 F F F 14 280-299 193 193 193 340 340 340 F F F 15 300-319 193 193 193 340 340 340 F F F 16 320-339 195 195 195 348 348 348 F F F 17 340-359 190 190 190 337 337 337 F F F 18 360-379 195 195 195 348 348 348 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 194 194 194 344 344 344 F F F 21 420-439 194 194 194 344 344 344 F F F 22 440-459 194 194 194 341 341 341 F F F 23 460-479 188 188 188 335 335 335 F F F 24 480-499 195 195 195 348 348 348 F F F 25 500-519 194 194 194 346 346 346 F F F 26 520-539 193 193 193 340 340 340 F F F 27 540-559 195 195 195 348 348 348 F F F 28 560-579 192 192 192 339 339 339 F F F 29 580-599 193 193 193 340 340 340 F F F Max gen: 195, Max flush: 348 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.944090946Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952264349Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952541321Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952572959Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952581848Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952593837Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Crucible a1079c8f-78e3-4736-ab68-a092aab7a859 has session id: cee240ab-1969-4719-8959-205ddc5f4cec","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952939024Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Upstairs opts: Upstairs UUID: a1079c8f-78e3-4736-ab68-a092aab7a859, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952976192Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Crucible stats registered with UUID: a1079c8f-78e3-4736-ab68-a092aab7a859","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.952987251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:19.952996631Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953130742Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.95316049Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953171699Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953275962Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953286232Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"a1079c8f-78e3-4736-ab68-a092aab7a859 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.95330373Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.95331256Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"0","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953322159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"1","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953332009Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"2","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953341598Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953364087Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953377916Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"ds_connection connected from Ok(127.0.0.1:53961)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953418193Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"ds_connection connected from Ok(127.0.0.1:62298)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953431502Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56297)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.953513207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954094648Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"0","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954114907Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"1","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954199582Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"2","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954434266Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954462474Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[0]R flush_numbers[0..12]: [338, 338, 339, 339, 339, 334, 344, 339, 336, 347, 343, 348]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954509151Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[0]R generation[0..12]: [191, 191, 192, 192, 192, 186, 194, 192, 189, 194, 194, 195]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.95452112Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.95453254Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[1]R flush_numbers[0..12]: [338, 338, 339, 339, 339, 334, 344, 339, 336, 347, 343, 348]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954543279Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[1]R generation[0..12]: [191, 191, 192, 192, 192, 186, 194, 192, 189, 194, 194, 195]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954553948Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954565038Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[2]R flush_numbers[0..12]: [338, 338, 339, 339, 339, 334, 344, 339, 336, 347, 343, 348]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954575817Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[2]R generation[0..12]: [191, 191, 192, 192, 192, 186, 194, 192, 189, 194, 194, 195]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954588776Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954599815Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954609285Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954618974Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954628423Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954637773Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"":"downstairs","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954647502Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954656312Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"0","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954666011Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"1","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.95467581Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"client":"2","":"downstairs","session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.9546857Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954695159Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"a1079c8f-78e3-4736-ab68-a092aab7a859 is now active with session: cee240ab-1969-4719-8959-205ddc5f4cec","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954705198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954713918Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"session_id":"cee240ab-1969-4719-8959-205ddc5f4cec"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954723017Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Activated sub_volume a1079c8f-78e3-4736-ab68-a092aab7a859","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954731137Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.954740726Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.955054945Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"task":"crutest"} 01/30 Write block 98 len 28672 data: 13 15 16 17 18 16 13 02/30 Read block 541 len 40960 03/30 Flush 04/30 Write block 465 len 4096 data: 11 05/30 Write block 90 len 36864 data: 15 16 16 16 15 13 13 10 14 06/30 Read block 361 len 40960 07/30 Write block 179 len 28672 data: 18 19 17 18 15 17 18 08/30 Read block 315 len 36864 09/30 Write block 458 len 12288 data: 14 13 12 10/30 Read block 80 len 24576 11/30 Read block 419 len 28672 12/30 Write block 228 len 32768 data: 16 15 14 12 10 13 12 18 13/30 Read block 83 len 4096 14/30 Read block 121 len 28672 15/30 Write block 278 len 20480 data: 7 10 8 11 12 16/30 Read block 16 len 36864 17/30 Read block 377 len 20480 18/30 Read block 128 len 24576 19/30 Read block 347 len 12288 20/30 Read block 332 len 20480 21/30 Read block 147 len 40960 22/30 Read block 21 len 12288 23/30 Read block 31 len 24576 24/30 Write block 303 len 40960 data: 18 19 19 17 18 17 15 19 15 15 25/30 Flush 26/30 Write block 139 len 40960 data: 15 13 13 18 18 17 17 16 22 23 27/30 Read block 256 len 36864 28/30 Write block 518 len 4096 data: 15 29/30 Read block 292 len 20480 30/30 Write block 510 len 32768 data: 14 16 15 16 17 22 23 21 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 26 26 0 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:19.965230706Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:19.965446552Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1923,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 196 196 196 350 350 349 F F F <--- 5 100-119 196 196 196 349 349 349 F F F 6 120-139 194 194 194 344 344 344 T T F <--- 7 140-159 192 192 192 339 339 339 T T F <--- 8 160-179 196 196 189 350 350 336 F F F <--- 9 180-199 196 196 194 350 350 347 F F F <--- 10 200-219 194 194 194 343 343 343 F F F 11 220-239 196 196 195 350 350 348 F F F <--- 12 240-259 194 194 194 344 344 344 F F F 13 260-279 196 196 194 350 350 344 F F F <--- 14 280-299 196 196 193 350 350 340 F F F <--- 15 300-319 196 196 193 350 350 340 F F F <--- 16 320-339 195 195 195 348 348 348 F F F 17 340-359 190 190 190 337 337 337 F F F 18 360-379 195 195 195 348 348 348 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 194 194 194 344 344 344 F F F 21 420-439 194 194 194 344 344 344 F F F 22 440-459 196 196 194 350 350 341 F F F <--- 23 460-479 196 196 188 350 350 335 F F F <--- 24 480-499 195 195 195 348 348 348 F F F 25 500-519 194 194 194 346 346 346 T T F <--- 26 520-539 193 193 193 340 340 340 F F F 27 540-559 195 195 195 348 348 348 F F F 28 560-579 192 192 192 339 339 339 F F F 29 580-599 193 193 193 340 340 340 F F F Max gen: 196, Max flush: 350 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 196 196 196 350 350 350 F F F 5 100-119 196 196 196 349 349 349 F F F 6 120-139 197 197 197 351 351 351 F F F 7 140-159 197 197 197 351 351 351 F F F 8 160-179 196 196 196 350 350 350 F F F 9 180-199 196 196 196 350 350 350 F F F 10 200-219 194 194 194 343 343 343 F F F 11 220-239 196 196 196 350 350 350 F F F 12 240-259 194 194 194 344 344 344 F F F 13 260-279 196 196 196 350 350 350 F F F 14 280-299 196 196 196 350 350 350 F F F 15 300-319 196 196 196 350 350 350 F F F 16 320-339 195 195 195 348 348 348 F F F 17 340-359 190 190 190 337 337 337 F F F 18 360-379 195 195 195 348 348 348 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 194 194 194 344 344 344 F F F 21 420-439 194 194 194 344 344 344 F F F 22 440-459 196 196 196 350 350 350 F F F 23 460-479 196 196 196 350 350 350 F F F 24 480-499 195 195 195 348 348 348 F F F 25 500-519 197 197 197 351 351 351 F F F 26 520-539 193 193 193 340 340 340 F F F 27 540-559 195 195 195 348 348 348 F F F 28 560-579 192 192 192 339 339 339 F F F 29 580-599 193 193 193 340 340 340 F F F Max gen: 197, Max flush: 351 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.218814655Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22709988Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227382222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"72d844bb3d3514e8d24f68105afb5f4d6739b4cc\",\n git_commit_timestamp: \"2026-09-18T03:11:39.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22740735Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22741615Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227428139Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Crucible a88d8ec2-babe-4565-97f5-70d349bc5d32 has session id: 6c5092db-3c24-4710-b01f-02351ec2ee10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227772296Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Upstairs opts: Upstairs UUID: a88d8ec2-babe-4565-97f5-70d349bc5d32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227800814Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Crucible stats registered with UUID: a88d8ec2-babe-4565-97f5-70d349bc5d32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227811944Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:24.227821573Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227950544Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227965863Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.227978553Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228133222Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228149201Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"a88d8ec2-babe-4565-97f5-70d349bc5d32 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228183339Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228192968Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"0","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228202678Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"1","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228212327Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"2","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228221917Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228232136Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228242035Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"ds_connection connected from Ok(127.0.0.1:56400)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228258204Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"ds_connection connected from Ok(127.0.0.1:59141)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22832698Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"ds_connection connected from Ok(127.0.0.1:33744)","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228338459Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228919801Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"1","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.228940769Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"0","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229015444Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"2","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229247749Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229281397Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[0]R flush_numbers[0..12]: [338, 338, 339, 339, 350, 349, 351, 351, 350, 350, 343, 350]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229345513Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[0]R generation[0..12]: [191, 191, 192, 192, 196, 196, 197, 197, 196, 196, 194, 196]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229356952Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229368051Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[1]R flush_numbers[0..12]: [338, 338, 339, 339, 350, 349, 351, 351, 350, 350, 343, 350]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22937895Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[1]R generation[0..12]: [191, 191, 192, 192, 196, 196, 197, 197, 196, 196, 194, 196]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22938972Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229400809Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[2]R flush_numbers[0..12]: [338, 338, 339, 339, 350, 349, 351, 351, 350, 350, 343, 350]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229411518Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[2]R generation[0..12]: [191, 191, 192, 192, 196, 196, 197, 197, 196, 196, 194, 196]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229422198Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229433167Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229442576Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229452316Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229461605Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229472574Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"":"downstairs","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229482244Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229491103Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"0","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229500672Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"1","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229510372Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"client":"2","":"downstairs","session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229520231Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229529251Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"a88d8ec2-babe-4565-97f5-70d349bc5d32 is now active with session: 6c5092db-3c24-4710-b01f-02351ec2ee10","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.22953925Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229548079Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"session_id":"6c5092db-3c24-4710-b01f-02351ec2ee10"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229557419Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Activated sub_volume a88d8ec2-babe-4565-97f5-70d349bc5d32","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229565478Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229573738Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.229895347Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"task":"crutest"} 01/30 Write block 254 len 40960 data: 13 10 10 10 10 9 10 12 10 9 02/30 Flush 03/30 Write block 291 len 28672 data: 15 11 11 13 13 11 11 04/30 Write block 95 len 28672 data: 14 14 11 15 16 17 18 05/30 Write block 443 len 32768 data: 12 12 15 15 16 21 21 18 06/30 Read block 332 len 20480 07/30 Read block 524 len 40960 08/30 Read block 458 len 36864 09/30 Read block 221 len 4096 10/30 Write block 498 len 16384 data: 19 19 14 14 11/30 Write block 360 len 24576 data: 11 12 14 18 20 19 12/30 Read block 198 len 40960 13/30 Flush 14/30 Write block 157 len 36864 data: 19 20 19 17 17 12 10 8 10 15/30 Read block 191 len 40960 16/30 Read block 30 len 28672 17/30 Write block 24 len 32768 data: 16 13 16 14 16 14 13 13 18/30 Read block 318 len 40960 19/30 Read block 310 len 8192 20/30 Read block 5 len 28672 21/30 Read block 193 len 8192 22/30 Write block 573 len 16384 data: 10 11 12 13 23/30 Write block 343 len 32768 data: 14 14 16 17 18 16 14 15 24/30 Read block 362 len 24576 25/30 Read block 441 len 36864 26/30 Flush 27/30 Write block 88 len 36864 data: 16 15 16 17 17 17 16 15 15 28/30 Write block 449 len 36864 data: 22 19 18 17 19 19 16 15 13 29/30 Read block 361 len 24576 30/30 Read block 526 len 12288 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-18T03:35:24.239624207Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-18T03:35:24.239806195Z","hostname":"w-01M2S8TQGWWRE0ZVJH7Z4K0NTJ","pid":1931,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 198 198 191 354 354 338 F F F <--- 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 198 198 196 353 353 350 T T F <--- 5 100-119 198 198 196 353 353 349 F F F <--- 6 120-139 197 197 197 351 351 351 F F F 7 140-159 198 198 197 354 354 351 F F F <--- 8 160-179 198 198 196 354 354 350 F F F <--- 9 180-199 196 196 196 350 350 350 F F F 10 200-219 194 194 194 343 343 343 F F F 11 220-239 196 196 196 350 350 350 F F F 12 240-259 198 198 194 352 352 344 F F F <--- 13 260-279 198 198 196 352 352 350 F F F <--- 14 280-299 198 198 196 353 353 350 F F F <--- 15 300-319 196 196 196 350 350 350 F F F 16 320-339 195 195 195 348 348 348 F F F 17 340-359 198 198 190 354 354 337 F F F <--- 18 360-379 198 198 195 353 353 348 F F F <--- 19 380-399 190 190 190 337 337 337 F F F 20 400-419 194 194 194 344 344 344 F F F 21 420-439 194 194 194 344 344 344 F F F 22 440-459 198 198 196 353 353 350 T T F <--- 23 460-479 196 196 196 350 350 350 F F F 24 480-499 198 198 195 353 353 348 F F F <--- 25 500-519 198 198 197 353 353 351 F F F <--- 26 520-539 193 193 193 340 340 340 F F F 27 540-559 195 195 195 348 348 348 F F F 28 560-579 198 198 192 354 354 339 F F F <--- 29 580-599 193 193 193 340 340 340 F F F Max gen: 198, Max flush: 354 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 198 198 198 354 354 354 F F F 2 040-059 192 192 192 339 339 339 F F F 3 060-079 192 192 192 339 339 339 F F F 4 080-099 199 199 199 355 355 355 F F F 5 100-119 198 198 198 353 353 353 F F F 6 120-139 197 197 197 351 351 351 F F F 7 140-159 198 198 198 354 354 354 F F F 8 160-179 198 198 198 354 354 354 F F F 9 180-199 196 196 196 350 350 350 F F F 10 200-219 194 194 194 343 343 343 F F F 11 220-239 196 196 196 350 350 350 F F F 12 240-259 198 198 198 352 352 352 F F F 13 260-279 198 198 198 352 352 352 F F F 14 280-299 198 198 198 353 353 353 F F F 15 300-319 196 196 196 350 350 350 F F F 16 320-339 195 195 195 348 348 348 F F F 17 340-359 198 198 198 354 354 354 F F F 18 360-379 198 198 198 353 353 353 F F F 19 380-399 190 190 190 337 337 337 F F F 20 400-419 194 194 194 344 344 344 F F F 21 420-439 194 194 194 344 344 344 F F F 22 440-459 199 199 199 355 355 355 F F F 23 460-479 196 196 196 350 350 350 F F F 24 480-499 198 198 198 353 353 353 F F F 25 500-519 198 198 198 353 353 353 F F F 26 520-539 193 193 193 340 340 340 F F F 27 540-559 195 195 195 348 348 348 F F F 28 560-579 198 198 198 354 354 354 F F F 29 580-599 193 193 193 340 340 340 F F F Max gen: 199, Max flush: 355 7:28 Test duration Test completed real 7:28.022422798 user 25.817666550 sys 14.532432674 trap 0.020735244 tflt 0.065042032 dflt 0.074632969 kflt 0.003669934 lock 11:52:03.546917209 slp 36:07.912880114 lat 2.451050207 stop 0.019827801