Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.238944 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.015288253 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.013539909 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1135 1136 1137 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.817999982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.831571886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.83277786Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.8328565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.832867209Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.832880717Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"Crucible 19a5b790-1028-41d2-b8f4-f36c9c1ab075 has session id: 5bc8a518-15be-4ba0-988d-7da076382d2b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.833769582Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Upstairs opts: Upstairs UUID: 19a5b790-1028-41d2-b8f4-f36c9c1ab075, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.833882558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Crucible stats registered with UUID: 19a5b790-1028-41d2-b8f4-f36c9c1ab075","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.833898436Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:25.833918873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.834540003Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"2","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.834594675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"1","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.834606364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"0","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835070464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835113498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"19a5b790-1028-41d2-b8f4-f36c9c1ab075 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835126287Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835140925Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"0","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835160812Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"1","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835171871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"2","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835186599Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"2","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835198187Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"0","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835212686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"1","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59726)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835283786Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"0","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38179)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835314162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"2","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46111)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.835629412Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"io task","client":"1","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.837382035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"1","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.837398463Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"0","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.837413791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"2","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838173513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838321613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838334222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.83834696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838359019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838370807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838383475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838395254Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838407082Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838419791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838430709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838441978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838452676Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838463495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"":"downstairs","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838476373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838486572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"0","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838498191Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"1","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838509619Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"client":"2","":"downstairs","session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838521378Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838532106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"19a5b790-1028-41d2-b8f4-f36c9c1ab075 is now active with session: 5bc8a518-15be-4ba0-988d-7da076382d2b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838543665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838553913Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"session_id":"5bc8a518-15be-4ba0-988d-7da076382d2b"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838564572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"Activated sub_volume 19a5b790-1028-41d2-b8f4-f36c9c1ab075","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838573831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.83858325Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.838880231Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- ,"time":"2026-05-14T20:46:25.885561343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} {"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30," Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 time"States::" Active2026-05-14T20:46:25.885599558Z" Active Active, "JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest" 1004 Acked Read 88} Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:25.885694676Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.88627064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.886297796Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:29.886308495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1140} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.91108331Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:29.919060178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.91912797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.926845651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927139124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927172079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927181658Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927194386Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Crucible bfd0e27b-59d1-409f-84c3-e0958bae85d2 has session id: 531cf025-a555-4148-b10a-75044e681b34","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927581036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Upstairs opts: Upstairs UUID: bfd0e27b-59d1-409f-84c3-e0958bae85d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927638429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Crucible stats registered with UUID: bfd0e27b-59d1-409f-84c3-e0958bae85d2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927652657Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:29.927662716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927794279Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"0","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927812696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"1","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.927823825Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"2","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928072323Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928089511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"bfd0e27b-59d1-409f-84c3-e0958bae85d2 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928134155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928144533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"0","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928155352Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"1","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928165951Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"2","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928176489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"2","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928187558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"0","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928198407Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"1","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:56491)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928283486Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"0","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:53357)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928352177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"2","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:39371)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.928449834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"io task","client":"1","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929047987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"1","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929079563Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"0","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929109489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"2","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929453774Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929490779Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929502818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929515196Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929526585Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929537933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929549862Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.92956116Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929572609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929584827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929597086Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929607894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929618163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929628542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"":"downstairs","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.92963924Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929648969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"0","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929659927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"1","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929670856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"client":"2","":"downstairs","session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929682045Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929692163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"bfd0e27b-59d1-409f-84c3-e0958bae85d2 is now active with session: 531cf025-a555-4148-b10a-75044e681b34","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929703112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929712811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"session_id":"531cf025-a555-4148-b10a-75044e681b34"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929722879Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Activated sub_volume bfd0e27b-59d1-409f-84c3-e0958bae85d2","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929731708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.929740727Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.930468353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} 01/30 Write block 440 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 367 len 4096 03/30 Write block 457 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Write block 45 len 40960 data: 2 2 2 2 2 2 2 2 2 2 05/30 Read block 419 len 40960 06/30 Write block 51 len 12288 data: 3 3 3 07/30 Write block 242 len 20480 data: 2 2 2 2 2 08/30 Write block 389 len 40960 data: 2 2 2 2 2 2 2 2 2 2 09/30 Read block 162 len 36864 10/30 Read block 383 len 32768 11/30 Write block 511 len 28672 data: 2 2 2 2 2 2 2 12/30 Flush 13/30 Write block 119 len 32768 data: 2 2 2 2 2 2 2 2 14/30 Write block 41 len 4096 data: 2 15/30 Write block 192 len 8192 data: 2 2 16/30 Write block 528 len 40960 data: 2 2 2 2 2 2 2 2 2 2 17/30 Write block 430 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Flush 19/30 Flush 20/30 Write block 558 len 20480 data: 2 2 2 2 2 21/30 Write block 0 len 12288 data: 2 2 2 22/30 Write block 363 len 4096 data: 2 23/30 Write block 390 len 8192 data: 3 3 24/30 Flush 25/30 Write block 517 len 24576 data: 3 2 2 2 2 2 26/30 Write block 459 len 12288 data: 3 3 3 27/30 Write block 458 len 40960 data: 3 4 4 4 3 3 3 2 2 2 28/30 Read block 150 len 12288 29/30 Read block 50 len 16384 30/30 Write block 122 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:29.939567326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:29.939772959Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1142,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 2 2 1 5 5 F F F <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 2 2 1 3 3 F F F <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 2 2 1 3 3 F F F <--- 6 120-139 1 2 2 1 3 3 F T T <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 2 2 1 3 3 F F F <--- 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 2 2 1 2 2 F F F <--- 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 2 2 1 5 5 F F F <--- 19 380-399 1 2 2 1 5 5 F F F <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 2 2 1 3 3 F F F <--- 22 440-459 1 2 2 1 2 2 T T T <--- 23 460-479 1 2 2 1 2 2 F T T <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 2 2 1 2 2 F T T <--- 26 520-539 1 2 2 1 3 3 F T T <--- 27 540-559 1 2 2 1 5 5 F F F <--- 28 560-579 1 2 2 1 5 5 F F F <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 5 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 3 3 3 6 6 6 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 3 3 3 6 6 6 F F F 23 460-479 3 3 3 6 6 6 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 6 6 6 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 2 2 2 5 5 5 F F F 28 560-579 2 2 2 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.358645369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:44.366565845Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.366627627Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.374459744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.374767724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.374807549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.374817648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.374830426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Crucible d206b094-36c7-4425-b991-a2769062199e has session id: 6e8ef14d-afcb-4213-a5ad-12692c199c41","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375221555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Upstairs opts: Upstairs UUID: d206b094-36c7-4425-b991-a2769062199e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375253321Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Crucible stats registered with UUID: d206b094-36c7-4425-b991-a2769062199e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.37526557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:44.375275988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.37542031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375439327Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375452326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375621954Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375636692Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"d206b094-36c7-4425-b991-a2769062199e active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375676787Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375687115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"0","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375697854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"1","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375708512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"2","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375719011Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.3757301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375741408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"ds_connection connected from Ok(127.0.0.1:42927)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375828347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"ds_connection connected from Ok(127.0.0.1:60820)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375856863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"ds_connection connected from Ok(127.0.0.1:60127)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.375869582Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.376529886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"1","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.376544644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"2","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.37665848Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"0","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377026032Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[0]R flush_numbers[0..12]: [5, 1, 3, 1, 1, 3, 6, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377048729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[0]R generation[0..12]: [2, 1, 2, 1, 1, 2, 3, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377060708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377073256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[1]R flush_numbers[0..12]: [5, 1, 3, 1, 1, 3, 6, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377084994Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[1]R generation[0..12]: [2, 1, 2, 1, 1, 2, 3, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377096503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377108891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[2]R flush_numbers[0..12]: [5, 1, 3, 1, 1, 3, 6, 1, 1, 3, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.37712034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[2]R generation[0..12]: [2, 1, 2, 1, 1, 2, 3, 1, 1, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377131918Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377144097Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377154725Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377165894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377176503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377187001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"":"downstairs","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.37719798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377209648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"0","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377220477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"1","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377231316Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"client":"2","":"downstairs","session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377242324Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377252583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"d206b094-36c7-4425-b991-a2769062199e is now active with session: 6e8ef14d-afcb-4213-a5ad-12692c199c41","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377263761Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.37727372Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"session_id":"6e8ef14d-afcb-4213-a5ad-12692c199c41"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377283999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Activated sub_volume d206b094-36c7-4425-b991-a2769062199e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377293128Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377302256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.377672568Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} 01/30 Write block 266 len 20480 data: 2 2 2 2 2 02/30 Read block 378 len 20480 03/30 Write block 129 len 12288 data: 2 2 2 04/30 Write block 10 len 12288 data: 2 2 2 05/30 Write block 446 len 12288 data: 3 2 2 06/30 Write block 136 len 20480 data: 2 2 2 2 2 07/30 Write block 573 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Read block 390 len 8192 09/30 Read block 107 len 4096 10/30 Read block 72 len 20480 11/30 Write block 182 len 20480 data: 2 2 2 2 2 12/30 Write block 183 len 16384 data: 3 3 3 3 13/30 Write block 331 len 24576 data: 2 2 2 2 2 2 14/30 Write block 446 len 12288 data: 4 3 3 15/30 Read block 305 len 36864 16/30 Read block 230 len 28672 17/30 Read block 105 len 4096 18/30 Write block 286 len 40960 data: 2 2 2 2 2 2 2 2 2 2 19/30 Write block 197 len 28672 data: 2 2 2 2 2 2 2 20/30 Read block 78 len 4096 21/30 Write block 222 len 16384 data: 2 2 2 2 22/30 Write block 389 len 36864 data: 3 4 4 3 3 3 3 3 3 23/30 Write block 357 len 32768 data: 2 2 2 2 2 2 3 2 24/30 Flush 25/30 Write block 504 len 36864 data: 2 2 2 2 2 2 2 3 3 26/30 Write block 220 len 24576 data: 2 2 3 3 3 3 27/30 Write block 538 len 24576 data: 2 2 2 2 2 2 28/30 Read block 284 len 24576 29/30 Write block 90 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Write block 590 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 19 2 2 23 Done 11 28 28 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:44.386310181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:44.386528253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1151,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 4 4 5 7 7 T F F <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F T T <--- 5 100-119 2 2 2 3 3 3 F F F 6 120-139 3 4 4 6 7 7 T F F <--- 7 140-159 1 4 4 1 7 7 T F F <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 4 4 3 7 7 T F F <--- 10 200-219 1 4 4 1 7 7 F F F <--- 11 220-239 1 4 4 1 7 7 F T T <--- 12 240-259 2 2 2 2 2 2 F F F 13 260-279 1 4 4 1 7 7 T F F <--- 14 280-299 1 4 4 1 7 7 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 4 4 1 7 7 F F F <--- 17 340-359 1 4 4 1 7 7 F F F <--- 18 360-379 2 4 4 5 7 7 F F F <--- 19 380-399 2 4 4 5 7 7 F F F <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 3 4 4 6 7 7 T F F <--- 23 460-479 3 3 3 6 6 6 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 6 6 6 F T T <--- 26 520-539 3 3 3 6 6 6 F T T <--- 27 540-559 2 2 2 5 5 5 F T T <--- 28 560-579 2 4 4 5 7 7 T F F <--- 29 580-599 1 4 4 1 7 7 T T T <--- Max gen: 4, Max flush: 7 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 4 4 4 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 4 4 4 7 7 7 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 4 4 4 7 7 7 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 4 4 4 7 7 7 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 3 3 3 6 6 6 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 5 5 5 8 8 8 F F F 26 520-539 5 5 5 8 8 8 F F F 27 540-559 5 5 5 8 8 8 F F F 28 560-579 4 4 4 7 7 7 F F F 29 580-599 5 5 5 8 8 8 F F F Max gen: 5, Max flush: 8 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.675803336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.685155886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.685485353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.685520659Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.685533027Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.685546675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Crucible f9118db1-796c-4b7d-aa1a-a13846d392c4 has session id: 5defac07-45c6-49a4-838c-6daccfc0feb8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.685989708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Upstairs opts: Upstairs UUID: f9118db1-796c-4b7d-aa1a-a13846d392c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686035022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Crucible stats registered with UUID: f9118db1-796c-4b7d-aa1a-a13846d392c4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.68604811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:58.686059359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686197651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"1","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686233296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"0","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686246135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"2","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686405474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686420992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"f9118db1-796c-4b7d-aa1a-a13846d392c4 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686449019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686459537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"0","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686470976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"1","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686482254Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"2","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686495723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"2","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686507491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"0","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686519299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"1","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36113)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686583731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"2","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:35502)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686596519Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"1","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61231)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.686608668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"io task","client":"0","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687318036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"1","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687370819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"0","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687512421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"2","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687821281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[0]R flush_numbers[0..12]: [7, 1, 3, 1, 8, 3, 7, 7, 1, 7, 7, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687843668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[0]R generation[0..12]: [4, 1, 2, 1, 5, 2, 4, 4, 1, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687855957Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687869035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[1]R flush_numbers[0..12]: [7, 1, 3, 1, 8, 3, 7, 7, 1, 7, 7, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687881263Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[1]R generation[0..12]: [4, 1, 2, 1, 5, 2, 4, 4, 1, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687893192Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.68790849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[2]R flush_numbers[0..12]: [7, 1, 3, 1, 8, 3, 7, 7, 1, 7, 7, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687920918Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[2]R generation[0..12]: [4, 1, 2, 1, 5, 2, 4, 4, 1, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687932767Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687945655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687956664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687968112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687979221Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.687990319Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"":"downstairs","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688002068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688012726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"0","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688024475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"1","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688036203Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"client":"2","":"downstairs","session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688048092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.68805899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"f9118db1-796c-4b7d-aa1a-a13846d392c4 is now active with session: 5defac07-45c6-49a4-838c-6daccfc0feb8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688070869Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688082367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"session_id":"5defac07-45c6-49a4-838c-6daccfc0feb8"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688093066Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Activated sub_volume f9118db1-796c-4b7d-aa1a-a13846d392c4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688102455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688112023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.688474517Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"task":"crutest"} 01/30 Write block 122 len 4096 data: 4 02/30 Read block 459 len 24576 03/30 Read block 486 len 12288 04/30 Flush 05/30 Write block 40 len 12288 data: 2 3 2 06/30 Write block 577 len 16384 data: 3 3 3 3 07/30 Read block 352 len 8192 08/30 Flush 09/30 Write block 549 len 8192 data: 2 2 10/30 Flush 11/30 Write block 386 len 16384 data: 2 2 2 4 12/30 Read block 366 len 28672 13/30 Write block 511 len 12288 data: 4 4 3 14/30 Write block 186 len 8192 data: 4 2 15/30 Write block 467 len 36864 data: 3 2 2 2 2 2 2 2 2 16/30 Write block 350 len 8192 data: 2 2 17/30 Read block 208 len 4096 18/30 Read block 174 len 8192 19/30 Flush 20/30 Write block 15 len 16384 data: 2 2 2 2 21/30 Read block 170 len 16384 22/30 Read block 418 len 16384 23/30 Read block 20 len 20480 24/30 Read block 124 len 28672 25/30 Write block 589 len 20480 data: 2 3 3 3 3 26/30 Flush 27/30 Write block 52 len 4096 data: 4 28/30 Read block 162 len 20480 29/30 Read block 172 len 32768 30/30 Write block 139 len 32768 data: 3 3 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:46:58.697915315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:46:58.698166943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1160,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 4 13 13 7 F F F <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 6 6 2 10 10 3 T T F <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 6 6 4 9 9 7 T T F <--- 7 140-159 4 4 4 7 7 7 T T F <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 4 12 12 7 F F F <--- 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 4 4 4 7 7 7 F F F 17 340-359 6 6 4 12 12 7 F F F <--- 18 360-379 4 4 4 7 7 7 F F F 19 380-399 6 6 4 12 12 7 F F F <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 6 6 3 12 12 6 F F F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 6 6 5 12 12 8 F F F <--- 26 520-539 5 5 5 8 8 8 F F F 27 540-559 6 6 5 11 11 8 F F F <--- 28 560-579 6 6 4 10 10 7 F F F <--- 29 580-599 6 6 5 13 13 8 F F F <--- Max gen: 6, Max flush: 13 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 13 13 13 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 7 7 7 14 14 14 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 7 7 7 14 14 14 F F F 7 140-159 7 7 7 14 14 14 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 8 8 8 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 4 4 4 7 7 7 F F F 17 340-359 6 6 6 12 12 12 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 6 6 6 12 12 12 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 6 6 6 12 12 12 F F F 26 520-539 5 5 5 8 8 8 F F F 27 540-559 6 6 6 11 11 11 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 7, Max flush: 14 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.973283176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.98214106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.98245104Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.982483706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.982493565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.982506513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Crucible 0e457844-7223-414a-b307-e455141ded85 has session id: 8f9231c4-89d7-4aed-9413-55a89d6d7cdb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.982864577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Upstairs opts: Upstairs UUID: 0e457844-7223-414a-b307-e455141ded85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.982936238Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Crucible stats registered with UUID: 0e457844-7223-414a-b307-e455141ded85","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.982964404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:12.982975852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983086848Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983108155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983119494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983269434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983288082Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"0e457844-7223-414a-b307-e455141ded85 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983308579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983318568Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"0","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983329307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"1","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983339785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"2","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983350474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983361483Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983372441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39420)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983422175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"ds_connection connected from Ok(127.0.0.1:38832)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983443592Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"ds_connection connected from Ok(127.0.0.1:63066)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.983514233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984083819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"0","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984199524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"1","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984225711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"2","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984527182Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[0]R flush_numbers[0..12]: [13, 1, 14, 1, 8, 3, 14, 14, 1, 12, 7, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984566137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[0]R generation[0..12]: [6, 1, 7, 1, 5, 2, 7, 7, 1, 6, 4, 5]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984578295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984590574Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[1]R flush_numbers[0..12]: [13, 1, 14, 1, 8, 3, 14, 14, 1, 12, 7, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984602102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[1]R generation[0..12]: [6, 1, 7, 1, 5, 2, 7, 7, 1, 6, 4, 5]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984613441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984625599Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[2]R flush_numbers[0..12]: [13, 1, 14, 1, 8, 3, 14, 14, 1, 12, 7, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984637008Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[2]R generation[0..12]: [6, 1, 7, 1, 5, 2, 7, 7, 1, 6, 4, 5]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984648286Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984660485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984671093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984681942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.98469247Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984704879Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"":"downstairs","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984715917Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984725766Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"0","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984736975Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"1","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984747973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"client":"2","":"downstairs","session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984759252Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.98476954Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"0e457844-7223-414a-b307-e455141ded85 is now active with session: 8f9231c4-89d7-4aed-9413-55a89d6d7cdb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984780669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984790318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"session_id":"8f9231c4-89d7-4aed-9413-55a89d6d7cdb"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984800356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Activated sub_volume 0e457844-7223-414a-b307-e455141ded85","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984809415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.984818474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.985123675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"task":"crutest"} 01/30 Write block 397 len 4096 data: 4 02/30 Write block 320 len 16384 data: 2 2 2 2 03/30 Write block 121 len 4096 data: 3 04/30 Write block 224 len 40960 data: 4 4 2 2 2 2 2 2 2 2 05/30 Read block 427 len 16384 06/30 Flush 07/30 Write block 538 len 20480 data: 3 3 3 3 3 08/30 Write block 23 len 20480 data: 2 2 2 2 2 09/30 Read block 460 len 4096 10/30 Read block 502 len 28672 11/30 Write block 139 len 16384 data: 4 4 3 3 12/30 Read block 507 len 36864 13/30 Read block 137 len 4096 14/30 Flush 15/30 Write block 325 len 4096 data: 2 16/30 Read block 353 len 36864 17/30 Read block 450 len 28672 18/30 Read block 562 len 28672 19/30 Write block 5 len 28672 data: 2 2 2 2 2 3 3 20/30 Read block 457 len 4096 21/30 Read block 472 len 32768 22/30 Read block 101 len 32768 23/30 Write block 377 len 24576 data: 2 2 2 2 2 2 24/30 Read block 47 len 12288 25/30 Write block 430 len 12288 data: 3 3 3 26/30 Read block 367 len 8192 27/30 Write block 555 len 12288 data: 2 2 2 28/30 Write block 306 len 8192 data: 2 2 29/30 Read block 271 len 20480 30/30 Read block 190 len 12288 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 0 27 Done 29 4 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:12.993786834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:12.993996937Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1169,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 13 13 13 T F T <--- 1 020-039 8 1 8 16 1 16 F F F <--- 2 040-059 7 7 7 14 14 14 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 7 8 16 14 16 F T F <--- 7 140-159 8 7 8 16 14 16 F F F <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 8 5 8 15 8 15 F T F <--- 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 1 1 1 1 1 1 T F T <--- 16 320-339 8 4 8 15 7 15 T T T <--- 17 340-359 6 6 6 12 12 12 F F F 18 360-379 4 4 4 7 7 7 T F T <--- 19 380-399 8 6 8 15 12 15 T T T <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 T F T <--- 22 440-459 4 4 4 7 7 7 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 6 6 6 12 12 12 F F F 26 520-539 8 5 8 16 8 16 F F F <--- 27 540-559 8 6 8 16 11 16 T F T <--- 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 8, Max flush: 16 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 17 17 17 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 7 7 7 14 14 14 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 8 8 8 16 16 16 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 9 9 9 17 17 17 F F F 16 320-339 9 9 9 17 17 17 F F F 17 340-359 6 6 6 12 12 12 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 9 9 9 17 17 17 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 6 6 6 12 12 12 F F F 26 520-539 8 8 8 16 16 16 F F F 27 540-559 9 9 9 17 17 17 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 9, Max flush: 17 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.270295378Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.279701451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280036858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280068144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280080272Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280093691Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Crucible a860412a-4cf6-4e6c-ad01-0e00dededdc9 has session id: 0a069fac-1602-4adf-b84c-4e09488e3301","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280540243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Upstairs opts: Upstairs UUID: a860412a-4cf6-4e6c-ad01-0e00dededdc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280574208Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Crucible stats registered with UUID: a860412a-4cf6-4e6c-ad01-0e00dededdc9","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280586847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:27.280597835Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280759744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"1","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.28079314Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"2","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280805988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"0","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280969987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280980726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"a860412a-4cf6-4e6c-ad01-0e00dededdc9 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.280999293Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281019821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"0","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281031309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"1","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281042458Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"2","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281056006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"2","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281067845Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"0","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281079393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"1","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"ds_connection connected from Ok(127.0.0.1:52332)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281170401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"1","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"ds_connection connected from Ok(127.0.0.1:35005)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281200117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"0","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"ds_connection connected from Ok(127.0.0.1:48088)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281226224Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"io task","client":"2","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.281969338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"0","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.2820294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"2","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282044078Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"1","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282394183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[0]R flush_numbers[0..12]: [17, 16, 14, 1, 8, 3, 16, 16, 1, 12, 7, 15]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.28241983Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[0]R generation[0..12]: [9, 8, 7, 1, 5, 2, 8, 8, 1, 6, 4, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282433328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282446346Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[1]R flush_numbers[0..12]: [17, 16, 14, 1, 8, 3, 16, 16, 1, 12, 7, 15]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282458505Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[1]R generation[0..12]: [9, 8, 7, 1, 5, 2, 8, 8, 1, 6, 4, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282473443Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282486291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[2]R flush_numbers[0..12]: [17, 16, 14, 1, 8, 3, 16, 16, 1, 12, 7, 15]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.28249828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[2]R generation[0..12]: [9, 8, 7, 1, 5, 2, 8, 8, 1, 6, 4, 8]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282510178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282522876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282533765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282545033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282555762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282566581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"":"downstairs","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282577959Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282588308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"0","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282599626Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"1","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282610975Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"client":"2","":"downstairs","session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282622403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282633332Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"a860412a-4cf6-4e6c-ad01-0e00dededdc9 is now active with session: 0a069fac-1602-4adf-b84c-4e09488e3301","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282644931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282656659Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"session_id":"0a069fac-1602-4adf-b84c-4e09488e3301"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282673347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Activated sub_volume a860412a-4cf6-4e6c-ad01-0e00dededdc9","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282682696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.282692154Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.283058997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"task":"crutest"} 01/30 Write block 328 len 32768 data: 2 2 2 3 3 3 3 3 02/30 Write block 492 len 12288 data: 2 2 2 03/30 Flush 04/30 Write block 218 len 40960 data: 2 2 3 3 4 4 5 5 3 3 05/30 Write block 321 len 32768 data: 3 3 3 2 3 2 2 3 06/30 Read block 472 len 36864 07/30 Write block 61 len 16384 data: 2 2 2 2 08/30 Write block 244 len 40960 data: 3 3 3 2 2 2 2 2 2 2 09/30 Write block 256 len 4096 data: 2 10/30 Flush 11/30 Write block 394 len 8192 data: 4 4 12/30 Write block 14 len 36864 data: 2 3 3 3 3 2 2 2 2 13/30 Read block 194 len 36864 14/30 Write block 362 len 36864 data: 3 4 3 2 2 2 2 2 2 15/30 Write block 330 len 28672 data: 3 4 4 4 4 4 3 16/30 Write block 536 len 32768 data: 3 3 4 4 4 4 4 3 17/30 Write block 313 len 24576 data: 2 2 2 2 2 2 18/30 Write block 511 len 28672 data: 5 5 4 3 3 3 4 19/30 Write block 517 len 40960 data: 5 3 3 3 3 3 2 2 2 2 20/30 Write block 250 len 16384 data: 3 3 3 3 21/30 Read block 125 len 4096 22/30 Write block 44 len 40960 data: 2 3 3 3 3 3 3 4 5 4 23/30 Read block 525 len 16384 24/30 Read block 6 len 32768 25/30 Read block 245 len 40960 26/30 Write block 379 len 8192 data: 3 3 27/30 Read block 582 len 40960 28/30 Write block 225 len 16384 data: 6 4 4 3 29/30 Write block 362 len 12288 data: 4 5 4 30/30 Write block 381 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 28 3 34 Done 27 2 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:27.292410487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:27.29262402Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1178,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 17 17 17 T F T <--- 1 020-039 8 8 8 16 16 16 T F T <--- 2 040-059 7 7 7 14 14 14 T F T <--- 3 060-079 10 1 10 19 1 19 F F F <--- 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 8 8 8 16 16 16 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 10 4 10 19 7 19 F F F <--- 11 220-239 10 8 10 19 15 19 T F T <--- 12 240-259 10 2 10 19 2 19 T F T <--- 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 9 9 9 17 17 17 T F T <--- 16 320-339 10 9 10 19 17 19 T T T <--- 17 340-359 6 6 6 12 12 12 F F F 18 360-379 9 9 9 17 17 17 T F T <--- 19 380-399 9 9 9 17 17 17 T F T <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 9 9 9 17 17 17 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 1 10 18 1 18 F T F <--- 25 500-519 6 6 6 12 12 12 T F T <--- 26 520-539 8 8 8 16 16 16 T F T <--- 27 540-559 9 9 9 17 17 17 T F T <--- 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 10, Max flush: 19 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 11 11 11 20 20 20 F F F 2 040-059 11 11 11 20 20 20 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 8 8 8 16 16 16 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 11 11 11 20 20 20 F F F 12 240-259 11 11 11 20 20 20 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 11 11 11 20 20 20 F F F 16 320-339 11 11 11 20 20 20 F F F 17 340-359 6 6 6 12 12 12 F F F 18 360-379 11 11 11 20 20 20 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 9 9 9 17 17 17 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 11 11 11 20 20 20 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 6 6 6 10 10 10 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 11, Max flush: 20 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.583508924Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.59212438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.59242964Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.592461966Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.592471665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.592484223Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Crucible dc3eafe5-b0ba-4793-aefc-bd15aec76269 has session id: e4ad327e-026c-4f4a-af24-a2e075240642","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.592913768Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Upstairs opts: Upstairs UUID: dc3eafe5-b0ba-4793-aefc-bd15aec76269, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.592944274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Crucible stats registered with UUID: dc3eafe5-b0ba-4793-aefc-bd15aec76269","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.592956322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:41.592968141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593102913Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593134979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593146857Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593298438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593312476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"dc3eafe5-b0ba-4793-aefc-bd15aec76269 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593353591Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593363779Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"0","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593374258Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"1","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593384697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"2","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593395125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593405894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593416753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"ds_connection connected from Ok(127.0.0.1:51480)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593473545Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"ds_connection connected from Ok(127.0.0.1:46125)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593485464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"ds_connection connected from Ok(127.0.0.1:54394)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.593549905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594202711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"1","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594229487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"2","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594311317Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"0","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.59459714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[0]R flush_numbers[0..12]: [20, 20, 20, 19, 8, 3, 16, 16, 1, 12, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594624616Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[0]R generation[0..12]: [11, 11, 11, 10, 5, 2, 8, 8, 1, 6, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594636575Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594648443Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[1]R flush_numbers[0..12]: [20, 20, 20, 19, 8, 3, 16, 16, 1, 12, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594659422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[1]R generation[0..12]: [11, 11, 11, 10, 5, 2, 8, 8, 1, 6, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.59467036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594682269Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[2]R flush_numbers[0..12]: [20, 20, 20, 19, 8, 3, 16, 16, 1, 12, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594693347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[2]R generation[0..12]: [11, 11, 11, 10, 5, 2, 8, 8, 1, 6, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594704186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594715865Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594725963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594738772Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.59474886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594758859Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"":"downstairs","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594769318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594778636Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"0","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594789005Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"1","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594799334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"client":"2","":"downstairs","session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594809942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594819641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"dc3eafe5-b0ba-4793-aefc-bd15aec76269 is now active with session: e4ad327e-026c-4f4a-af24-a2e075240642","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.59483035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594839759Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"session_id":"e4ad327e-026c-4f4a-af24-a2e075240642"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594849467Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Activated sub_volume dc3eafe5-b0ba-4793-aefc-bd15aec76269","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594858166Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.594866915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.595204641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"task":"crutest"} 01/30 Write block 156 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 561 len 24576 data: 3 3 2 2 2 2 04/30 Read block 1 len 8192 05/30 Write block 538 len 20480 data: 5 5 5 5 5 06/30 Flush 07/30 Flush 08/30 Write block 328 len 16384 data: 4 3 4 5 09/30 Read block 53 len 16384 10/30 Write block 545 len 16384 data: 2 2 2 2 11/30 Read block 18 len 8192 12/30 Write block 416 len 24576 data: 2 2 2 2 2 2 13/30 Write block 60 len 36864 data: 2 3 3 3 3 2 2 2 2 14/30 Write block 273 len 20480 data: 2 2 2 2 2 15/30 Read block 235 len 24576 16/30 Read block 246 len 16384 17/30 Read block 303 len 12288 18/30 Read block 181 len 20480 19/30 Read block 423 len 36864 20/30 Read block 223 len 36864 21/30 Flush 22/30 Write block 437 len 24576 data: 3 3 2 3 3 3 23/30 Write block 558 len 20480 data: 3 3 3 4 4 24/30 Write block 204 len 28672 data: 2 2 2 2 2 2 2 25/30 Write block 585 len 4096 data: 2 26/30 Write block 222 len 24576 data: 5 5 6 7 5 5 27/30 Read block 588 len 12288 28/30 Read block 160 len 20480 29/30 Write block 358 len 40960 data: 3 3 3 3 5 6 5 3 3 3 30/30 Read block 441 len 28672 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Read 2 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 29 30 Done 29 30 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:41.604816158Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:41.605029611Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1187,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 11 11 11 20 20 20 F F F 2 040-059 11 11 11 20 20 20 F F F 3 060-079 12 12 10 24 24 19 F F F <--- 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 12 12 8 21 21 16 F F T <--- 8 160-179 12 12 1 21 21 1 F F T <--- 9 180-199 6 6 6 12 12 12 F F F 10 200-219 10 10 10 19 19 19 T T F <--- 11 220-239 11 11 11 20 20 20 T T F <--- 12 240-259 11 11 11 20 20 20 F F F 13 260-279 12 12 4 24 24 7 F F F <--- 14 280-299 4 4 4 7 7 7 F F F 15 300-319 11 11 11 20 20 20 F F F 16 320-339 12 12 11 24 24 20 F F F <--- 17 340-359 6 6 6 12 12 12 T T F <--- 18 360-379 11 11 11 20 20 20 T T F <--- 19 380-399 11 11 11 20 20 20 F F F 20 400-419 12 12 1 24 24 1 F F F <--- 21 420-439 12 12 9 24 24 17 T T F <--- 22 440-459 4 4 4 7 7 7 T T F <--- 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 12 12 11 22 22 20 F F F <--- 27 540-559 12 12 11 24 24 20 T T F <--- 28 560-579 12 12 6 22 22 10 T T F <--- 29 580-599 6 6 6 13 13 13 T T F <--- Max gen: 12, Max flush: 24 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 11 11 11 20 20 20 F F F 2 040-059 11 11 11 20 20 20 F F F 3 060-079 12 12 12 24 24 24 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 12 12 12 21 21 21 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 13 13 13 25 25 25 F F F 11 220-239 13 13 13 25 25 25 F F F 12 240-259 11 11 11 20 20 20 F F F 13 260-279 12 12 12 24 24 24 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 11 11 11 20 20 20 F F F 16 320-339 12 12 12 24 24 24 F F F 17 340-359 13 13 13 25 25 25 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 12 12 12 24 24 24 F F F 21 420-439 13 13 13 25 25 25 F F F 22 440-459 13 13 13 25 25 25 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 12 12 12 22 22 22 F F F 27 540-559 13 13 13 25 25 25 F F F 28 560-579 13 13 13 25 25 25 F F F 29 580-599 13 13 13 25 25 25 F F F Max gen: 13, Max flush: 25 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.882455515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.891469799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.891782089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.891814135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.891823384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.891835892Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Crucible 8d35d88a-a37f-43fb-b3cf-b07ccda76c46 has session id: 8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892219542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Upstairs opts: Upstairs UUID: 8d35d88a-a37f-43fb-b3cf-b07ccda76c46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892250828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Crucible stats registered with UUID: 8d35d88a-a37f-43fb-b3cf-b07ccda76c46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892262597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:55.892272895Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892405888Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892439994Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892452102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892577736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892594834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"8d35d88a-a37f-43fb-b3cf-b07ccda76c46 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.89262569Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892638218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"0","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892648927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"1","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892659525Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"2","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892669864Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892680643Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892691401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"ds_connection connected from Ok(127.0.0.1:58969)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.89277938Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"ds_connection connected from Ok(127.0.0.1:35416)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892915722Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"ds_connection connected from Ok(127.0.0.1:41630)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.892937799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.893521144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"1","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.893534602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"0","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.893662296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"2","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.893970876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[0]R flush_numbers[0..12]: [20, 20, 20, 24, 8, 3, 16, 21, 21, 12, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894006731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[0]R generation[0..12]: [11, 11, 11, 12, 5, 2, 8, 12, 12, 6, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.89401885Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894032808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[1]R flush_numbers[0..12]: [20, 20, 20, 24, 8, 3, 16, 21, 21, 12, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894044146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[1]R generation[0..12]: [11, 11, 11, 12, 5, 2, 8, 12, 12, 6, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894055315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894067293Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[2]R flush_numbers[0..12]: [20, 20, 20, 24, 8, 3, 16, 21, 21, 12, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894078572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[2]R generation[0..12]: [11, 11, 11, 12, 5, 2, 8, 12, 12, 6, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.89408972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894101639Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894111768Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894122476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894132605Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894142904Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"":"downstairs","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894153472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894163171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"0","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.89417369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"1","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894184198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"client":"2","":"downstairs","session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894195967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894205855Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"8d35d88a-a37f-43fb-b3cf-b07ccda76c46 is now active with session: 8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894216674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894226093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"session_id":"8d2cbc3a-68ac-435a-8b6c-f1cf5c4fe037"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894235952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Activated sub_volume 8d35d88a-a37f-43fb-b3cf-b07ccda76c46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.89424462Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894253469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.894583617Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"task":"crutest"} 01/30 Write block 169 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 402 len 8192 data: 2 2 03/30 Write block 202 len 36864 data: 3 3 3 3 3 3 3 3 3 04/30 Read block 177 len 32768 05/30 Flush 06/30 Write block 559 len 40960 data: 4 4 5 5 3 3 3 3 2 2 07/30 Read block 463 len 24576 08/30 Read block 65 len 12288 09/30 Write block 431 len 16384 data: 4 4 3 3 10/30 Flush 11/30 Write block 226 len 12288 data: 6 6 4 12/30 Write block 326 len 8192 data: 3 3 13/30 Read block 408 len 36864 14/30 Read block 386 len 40960 15/30 Read block 563 len 36864 16/30 Read block 183 len 40960 17/30 Write block 420 len 24576 data: 3 3 2 2 2 2 18/30 Flush 19/30 Write block 43 len 12288 data: 2 3 4 20/30 Read block 292 len 8192 21/30 Flush 22/30 Write block 309 len 28672 data: 2 2 2 2 3 3 3 23/30 Flush 24/30 Write block 157 len 32768 data: 3 3 3 3 3 3 3 3 25/30 Write block 538 len 20480 data: 6 6 6 6 6 26/30 Read block 528 len 16384 27/30 Read block 268 len 16384 28/30 Read block 172 len 40960 29/30 Write block 589 len 36864 data: 3 4 4 4 4 3 3 3 3 30/30 Write block 82 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 10 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 23 27 Done 23 23 2 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:47:55.904041293Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:47:55.904232259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1196,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 11 11 11 20 20 20 F F F 2 040-059 14 14 11 29 29 20 F F F <--- 3 060-079 12 12 12 24 24 24 F F F 4 080-099 5 5 5 8 8 8 T T F <--- 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 12 12 12 21 21 21 T T F <--- 8 160-179 14 14 14 26 26 26 T T F <--- 9 180-199 6 6 6 12 12 12 F F F 10 200-219 14 14 14 26 26 26 F F F 11 220-239 14 14 13 28 28 25 F F F <--- 12 240-259 11 11 11 20 20 20 F F F 13 260-279 12 12 12 24 24 24 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 14 14 11 30 30 20 F F F <--- 16 320-339 14 14 12 28 28 24 F F F <--- 17 340-359 13 13 13 25 25 25 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 14 14 13 28 28 25 F F F <--- 22 440-459 13 13 13 25 25 25 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 12 12 12 22 22 22 T T F <--- 27 540-559 14 14 13 27 27 25 T T T <--- 28 560-579 14 14 13 27 27 25 F F T <--- 29 580-599 13 13 13 25 25 25 T T F <--- Max gen: 14, Max flush: 30 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 11 11 11 20 20 20 F F F 2 040-059 14 14 14 29 29 29 F F F 3 060-079 12 12 12 24 24 24 F F F 4 080-099 15 15 15 31 31 31 F F F 5 100-119 2 2 2 3 3 3 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 15 15 15 31 31 31 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 14 14 14 26 26 26 F F F 11 220-239 14 14 14 28 28 28 F F F 12 240-259 11 11 11 20 20 20 F F F 13 260-279 12 12 12 24 24 24 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 14 14 14 30 30 30 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 13 13 13 25 25 25 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 14 14 14 28 28 28 F F F 22 440-459 13 13 13 25 25 25 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 15 15 15 31 31 31 F F F 27 540-559 15 15 15 31 31 31 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 15 15 15 31 31 31 F F F Max gen: 15, Max flush: 31 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.178999527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.187741717Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188116808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188153223Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188163102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.18817536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Crucible 20023a3f-05e8-4a14-bcf3-3949d822be93 has session id: d3a2d910-b52a-441d-a29b-8e8694f9bdbd","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188481381Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Upstairs opts: Upstairs UUID: 20023a3f-05e8-4a14-bcf3-3949d822be93, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188511417Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Crucible stats registered with UUID: 20023a3f-05e8-4a14-bcf3-3949d822be93","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188528105Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:10.188538024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188668457Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188695353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188706482Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.18887369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188887618Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"20023a3f-05e8-4a14-bcf3-3949d822be93 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188924534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188934472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"0","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188944771Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"1","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.18895505Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"2","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188965178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188975577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.188985966Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:64155)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.189097281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44744)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.189123328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40904)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.18918593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.189818188Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"2","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.189831656Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"1","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.189854243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"0","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.19019306Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[0]R flush_numbers[0..12]: [20, 20, 29, 24, 31, 3, 16, 31, 31, 12, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190229455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[0]R generation[0..12]: [11, 11, 14, 12, 15, 2, 8, 15, 15, 6, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190240843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190252512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[1]R flush_numbers[0..12]: [20, 20, 29, 24, 31, 3, 16, 31, 31, 12, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.19026336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[1]R generation[0..12]: [11, 11, 14, 12, 15, 2, 8, 15, 15, 6, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190274099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190285528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[2]R flush_numbers[0..12]: [20, 20, 29, 24, 31, 3, 16, 31, 31, 12, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190296366Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[2]R generation[0..12]: [11, 11, 14, 12, 15, 2, 8, 15, 15, 6, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190307145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190318513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190330062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.19034034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190350109Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190359878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"":"downstairs","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190370327Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190379485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"0","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190389824Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"1","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190400233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"client":"2","":"downstairs","session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190410701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.19042047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"20023a3f-05e8-4a14-bcf3-3949d822be93 is now active with session: d3a2d910-b52a-441d-a29b-8e8694f9bdbd","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190430849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190439968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"session_id":"d3a2d910-b52a-441d-a29b-8e8694f9bdbd"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190449496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Activated sub_volume 20023a3f-05e8-4a14-bcf3-3949d822be93","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190458035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190466514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.190831887Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"task":"crutest"} 01/30 Write block 26 len 36864 data: 3 3 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 395 len 12288 data: 5 4 5 04/30 Read block 409 len 28672 05/30 Read block 29 len 8192 06/30 Read block 84 len 24576 07/30 Write block 546 len 40960 data: 3 3 3 3 3 2 2 2 2 3 08/30 Write block 45 len 12288 data: 5 4 4 09/30 Read block 135 len 20480 10/30 Read block 79 len 8192 11/30 Read block 130 len 28672 12/30 Read block 53 len 12288 13/30 Read block 565 len 24576 14/30 Read block 24 len 8192 15/30 Write block 174 len 20480 data: 3 3 3 2 2 16/30 Write block 220 len 16384 data: 4 4 6 6 17/30 Flush 18/30 Write block 437 len 20480 data: 4 4 3 4 4 19/30 Read block 124 len 24576 20/30 Flush 21/30 Flush 22/30 Write block 311 len 4096 data: 3 23/30 Read block 451 len 32768 24/30 Read block 16 len 32768 25/30 Flush 26/30 Write block 241 len 28672 data: 2 3 3 4 4 4 3 27/30 Write block 241 len 4096 data: 3 28/30 Write block 554 len 32768 data: 3 4 3 3 4 5 5 6 29/30 Write block 117 len 28672 data: 2 2 3 3 4 5 4 30/30 Read block 453 len 24576 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:10.199703199Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:10.199890065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 16 11 16 32 20 32 F T F <--- 2 040-059 16 14 16 33 29 33 F F F <--- 3 060-079 12 12 12 24 24 24 F F F 4 080-099 15 15 15 31 31 31 F F F 5 100-119 2 2 2 3 3 3 T F T <--- 6 120-139 8 8 8 16 16 16 T F T <--- 7 140-159 15 15 15 31 31 31 F F F 8 160-179 16 15 16 33 31 33 F F F <--- 9 180-199 6 6 6 12 12 12 F F F 10 200-219 14 14 14 26 26 26 F F F 11 220-239 16 14 16 33 28 33 F F F <--- 12 240-259 11 11 11 20 20 20 T F T <--- 13 260-279 12 12 12 24 24 24 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 16 14 16 36 30 36 F F F <--- 16 320-339 14 14 14 28 28 28 F F F 17 340-359 13 13 13 25 25 25 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 16 11 16 33 20 33 F F F <--- 20 400-419 14 14 14 26 26 26 F F F 21 420-439 16 14 16 34 28 34 F F F <--- 22 440-459 16 13 16 34 25 34 F F F <--- 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 15 15 15 31 31 31 F F F 27 540-559 16 15 16 33 31 33 T F T <--- 28 560-579 14 14 14 27 27 27 T F T <--- 29 580-599 15 15 15 31 31 31 F F F Max gen: 16, Max flush: 36 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 16 16 16 33 33 33 F F F 3 060-079 12 12 12 24 24 24 F F F 4 080-099 15 15 15 31 31 31 F F F 5 100-119 17 17 17 37 37 37 F F F 6 120-139 17 17 17 37 37 37 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 16 16 16 33 33 33 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 14 14 14 26 26 26 F F F 11 220-239 16 16 16 33 33 33 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 12 12 12 24 24 24 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 16 16 16 36 36 36 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 13 13 13 25 25 25 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 16 16 16 33 33 33 F F F 20 400-419 14 14 14 26 26 26 F F F 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 11 11 11 20 20 20 F F F 26 520-539 15 15 15 31 31 31 F F F 27 540-559 17 17 17 37 37 37 F F F 28 560-579 17 17 17 37 37 37 F F F 29 580-599 15 15 15 31 31 31 F F F Max gen: 17, Max flush: 37 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.464507397Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.473166737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.473478647Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.473539479Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.473555397Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.473568155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Crucible e8f0ed6a-d525-4626-bcdb-3b567d986a76 has session id: d8165f21-4256-498a-a9c3-0ab6af81cfc8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.473953875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Upstairs opts: Upstairs UUID: e8f0ed6a-d525-4626-bcdb-3b567d986a76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474014877Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Crucible stats registered with UUID: e8f0ed6a-d525-4626-bcdb-3b567d986a76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474035415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:24.474045463Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474182126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474199503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474210252Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474385589Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474399468Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"e8f0ed6a-d525-4626-bcdb-3b567d986a76 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474437183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474449031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"0","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.47445946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"1","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474470098Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"2","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474480327Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474490976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474501534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43960)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474554737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56786)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474566486Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:53132)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.474679561Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475239849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"0","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475260356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"1","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475349945Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"2","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475651276Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[0]R flush_numbers[0..12]: [20, 32, 33, 24, 31, 37, 37, 31, 33, 12, 26, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475687491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[0]R generation[0..12]: [11, 16, 16, 12, 15, 17, 17, 15, 16, 6, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475699399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475713818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[1]R flush_numbers[0..12]: [20, 32, 33, 24, 31, 37, 37, 31, 33, 12, 26, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475725176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[1]R generation[0..12]: [11, 16, 16, 12, 15, 17, 17, 15, 16, 6, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475736435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475748243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[2]R flush_numbers[0..12]: [20, 32, 33, 24, 31, 37, 37, 31, 33, 12, 26, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475759432Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[2]R generation[0..12]: [11, 16, 16, 12, 15, 17, 17, 15, 16, 6, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.47577074Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475782699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475792907Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475803466Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475813535Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475823813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"":"downstairs","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475834342Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475843901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"0","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475854349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"1","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475864788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"client":"2","":"downstairs","session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475876476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475886415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"e8f0ed6a-d525-4626-bcdb-3b567d986a76 is now active with session: d8165f21-4256-498a-a9c3-0ab6af81cfc8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475897234Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475906593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"session_id":"d8165f21-4256-498a-a9c3-0ab6af81cfc8"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475916501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Activated sub_volume e8f0ed6a-d525-4626-bcdb-3b567d986a76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.47592549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.475934459Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.476269476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"task":"crutest"} 01/30 Write block 49 len 4096 data: 4 02/30 Write block 194 len 12288 data: 2 2 2 03/30 Read block 444 len 36864 04/30 Read block 17 len 4096 05/30 Write block 537 len 8192 data: 4 7 06/30 Read block 181 len 8192 07/30 Read block 509 len 8192 08/30 Write block 94 len 28672 data: 3 3 3 3 3 3 2 09/30 Write block 517 len 28672 data: 6 4 4 4 4 4 3 10/30 Write block 124 len 8192 data: 4 3 11/30 Read block 78 len 24576 12/30 Read block 447 len 4096 13/30 Read block 162 len 12288 14/30 Write block 405 len 24576 data: 2 2 2 2 2 2 15/30 Read block 5 len 20480 16/30 Write block 267 len 36864 data: 3 3 3 3 2 2 3 3 3 17/30 Read block 271 len 4096 18/30 Write block 350 len 28672 data: 3 3 2 2 2 2 2 19/30 Read block 274 len 24576 20/30 Write block 74 len 8192 data: 2 2 21/30 Read block 76 len 16384 22/30 Flush 23/30 Write block 553 len 40960 data: 3 4 5 4 4 5 6 6 7 6 24/30 Write block 135 len 4096 data: 2 25/30 Read block 225 len 28672 26/30 Write block 379 len 20480 data: 4 4 4 4 2 27/30 Write block 207 len 12288 data: 4 4 4 28/30 Read block 506 len 24576 29/30 Write block 526 len 36864 data: 3 2 3 3 3 3 3 3 3 30/30 Read block 188 len 4096 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 2 31 Done 30 1 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:24.485460137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:24.485646223Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1214,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 16 18 38 33 38 F T F <--- 3 060-079 18 12 18 38 24 38 F F F <--- 4 080-099 18 15 18 38 31 38 F F F <--- 5 100-119 18 17 18 38 37 38 F F F <--- 6 120-139 18 17 18 38 37 38 T F T <--- 7 140-159 15 15 15 31 31 31 F F F 8 160-179 16 16 16 33 33 33 F F F 9 180-199 18 6 18 38 12 38 F F F <--- 10 200-219 14 14 14 26 26 26 T F T <--- 11 220-239 16 16 16 33 33 33 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 18 12 18 38 24 38 F F F <--- 14 280-299 4 4 4 7 7 7 F F F 15 300-319 16 16 16 36 36 36 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 18 13 18 38 25 38 F F F <--- 18 360-379 13 13 13 25 25 25 T F T <--- 19 380-399 16 16 16 33 33 33 T F T <--- 20 400-419 18 14 18 38 26 38 F F F <--- 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 18 11 18 38 20 38 F F F <--- 26 520-539 18 15 18 38 31 38 T F T <--- 27 540-559 17 17 17 37 37 37 T F T <--- 28 560-579 17 17 17 37 37 37 T F T <--- 29 580-599 15 15 15 31 31 31 F F F Max gen: 18, Max flush: 38 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 18 18 18 38 38 38 F F F 4 080-099 18 18 18 38 38 38 F F F 5 100-119 18 18 18 38 38 38 F F F 6 120-139 19 19 19 39 39 39 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 16 16 16 33 33 33 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 19 19 19 39 39 39 F F F 11 220-239 16 16 16 33 33 33 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 16 16 16 36 36 36 F F F 16 320-339 14 14 14 28 28 28 F F F 17 340-359 18 18 18 38 38 38 F F F 18 360-379 19 19 19 39 39 39 F F F 19 380-399 19 19 19 39 39 39 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 18 18 18 38 38 38 F F F 26 520-539 19 19 19 39 39 39 F F F 27 540-559 19 19 19 39 39 39 F F F 28 560-579 19 19 19 39 39 39 F F F 29 580-599 15 15 15 31 31 31 F F F Max gen: 19, Max flush: 39 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.769689962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:38.777532938Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.77759093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785249499Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785529253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785568878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785579067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785591985Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Crucible e7c01f3e-88c5-4ae4-95d2-5af4d638f29d has session id: eaf37d1e-37ec-4a51-9353-8d26b78cd7cc","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785957688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Upstairs opts: Upstairs UUID: e7c01f3e-88c5-4ae4-95d2-5af4d638f29d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785981715Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Crucible stats registered with UUID: e7c01f3e-88c5-4ae4-95d2-5af4d638f29d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.785992763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:38.786003002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786163781Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786196897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786209145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786348037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786358296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"e7c01f3e-88c5-4ae4-95d2-5af4d638f29d active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786398171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.78640859Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"0","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786419528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"1","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786430587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"2","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786441385Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786452764Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60752)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786476931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786491179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41134)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786547062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:46830)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.786629231Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787251181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"2","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787304824Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"0","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787317612Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"1","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787683095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[0]R flush_numbers[0..12]: [20, 32, 38, 38, 38, 38, 39, 31, 33, 38, 39, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787711611Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[0]R generation[0..12]: [11, 16, 18, 18, 18, 18, 19, 15, 16, 18, 19, 16]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787723789Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787736018Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[1]R flush_numbers[0..12]: [20, 32, 38, 38, 38, 38, 39, 31, 33, 38, 39, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787747666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[1]R generation[0..12]: [11, 16, 18, 18, 18, 18, 19, 15, 16, 18, 19, 16]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787759165Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787771543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[2]R flush_numbers[0..12]: [20, 32, 38, 38, 38, 38, 39, 31, 33, 38, 39, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787785501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[2]R generation[0..12]: [11, 16, 18, 18, 18, 18, 19, 15, 16, 18, 19, 16]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.78779713Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787809588Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787820167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787830996Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787841504Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787852153Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"":"downstairs","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787863001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.7878728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"0","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787883909Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"1","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787894917Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"client":"2","":"downstairs","session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787906206Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787916634Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"e7c01f3e-88c5-4ae4-95d2-5af4d638f29d is now active with session: eaf37d1e-37ec-4a51-9353-8d26b78cd7cc","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787927873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787937762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"session_id":"eaf37d1e-37ec-4a51-9353-8d26b78cd7cc"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.78794795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Activated sub_volume e7c01f3e-88c5-4ae4-95d2-5af4d638f29d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787958069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.787967118Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.788272818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} 01/30 Write block 592 len 16384 data: 5 5 4 4 02/30 Write block 94 len 24576 data: 4 4 4 4 4 4 03/30 Read block 428 len 20480 04/30 Read block 536 len 8192 05/30 Read block 164 len 28672 06/30 Write block 376 len 8192 data: 2 3 07/30 Flush 08/30 Write block 302 len 40960 data: 2 2 2 2 3 3 2 3 3 4 09/30 Read block 173 len 32768 10/30 Write block 362 len 28672 data: 6 7 6 4 4 4 3 11/30 Write block 327 len 32768 data: 4 5 4 5 6 5 5 5 12/30 Write block 213 len 4096 data: 2 13/30 Read block 428 len 24576 14/30 Write block 218 len 12288 data: 3 3 5 15/30 Read block 548 len 32768 16/30 Read block 305 len 20480 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 128 len 28672 data: 2 3 3 3 2 2 2 21/30 Read block 554 len 16384 22/30 Flush 23/30 Write block 374 len 12288 data: 2 2 3 24/30 Write block 307 len 24576 data: 4 3 4 4 5 3 25/30 Write block 5 len 36864 data: 3 3 3 3 3 4 4 3 2 26/30 Read block 161 len 28672 27/30 Write block 332 len 24576 data: 6 6 6 5 4 2 28/30 Write block 388 len 28672 data: 3 5 5 5 4 4 5 29/30 Write block 161 len 24576 data: 4 4 4 4 2 2 30/30 Read block 566 len 32768 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 0 27 Done 4 29 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:38.797825583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:38.798039665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1223,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F T T <--- 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 18 18 18 38 38 38 F F F 4 080-099 18 20 20 38 40 40 T F F <--- 5 100-119 18 18 18 38 38 38 F F F 6 120-139 19 20 20 39 44 44 F F F <--- 7 140-159 15 15 15 31 31 31 F F F 8 160-179 16 16 16 33 33 33 F T T <--- 9 180-199 18 18 18 38 38 38 F F F 10 200-219 19 20 20 39 41 41 F F F <--- 11 220-239 16 20 20 33 41 41 F F F <--- 12 240-259 17 17 17 37 37 37 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 16 20 20 36 41 41 F T T <--- 16 320-339 14 20 20 28 41 41 F T T <--- 17 340-359 18 18 18 38 38 38 F F F 18 360-379 19 20 20 39 41 41 F T T <--- 19 380-399 19 19 19 39 39 39 F T T <--- 20 400-419 18 18 18 38 38 38 F F F 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 18 18 18 38 38 38 F F F 26 520-539 19 19 19 39 39 39 F F F 27 540-559 19 19 19 39 39 39 F F F 28 560-579 19 19 19 39 39 39 F F F 29 580-599 15 20 20 31 40 40 T F F <--- Max gen: 20, Max flush: 44 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 45 45 45 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 18 18 18 38 38 38 F F F 4 080-099 20 20 20 40 40 40 F F F 5 100-119 18 18 18 38 38 38 F F F 6 120-139 20 20 20 44 44 44 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 20 20 20 41 41 41 F F F 11 220-239 20 20 20 41 41 41 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 18 18 18 38 38 38 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 21 21 21 45 45 45 F F F 16 320-339 21 21 21 45 45 45 F F F 17 340-359 18 18 18 38 38 38 F F F 18 360-379 21 21 21 45 45 45 F F F 19 380-399 21 21 21 45 45 45 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 16 16 16 34 34 34 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 10 10 10 18 18 18 F F F 25 500-519 18 18 18 38 38 38 F F F 26 520-539 19 19 19 39 39 39 F F F 27 540-559 19 19 19 39 39 39 F F F 28 560-579 19 19 19 39 39 39 F F F 29 580-599 20 20 20 40 40 40 F F F Max gen: 21, Max flush: 45 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.073728564Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.082698854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083022482Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083057918Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083067686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083080675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Crucible a26a7ac9-b368-4e9b-978c-ba50a0404c93 has session id: 1597d6aa-b777-4838-8fa4-ff2cb69cc377","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083530976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Upstairs opts: Upstairs UUID: a26a7ac9-b368-4e9b-978c-ba50a0404c93, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083550394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Crucible stats registered with UUID: a26a7ac9-b368-4e9b-978c-ba50a0404c93","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083561812Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:53.083572351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083709453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083728191Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083741649Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083936054Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083950542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"a26a7ac9-b368-4e9b-978c-ba50a0404c93 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083987997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.083998546Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"0","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084009455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"1","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084020383Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"2","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084031322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.08404241Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084072626Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"ds_connection connected from Ok(127.0.0.1:45755)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084099253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"ds_connection connected from Ok(127.0.0.1:51828)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084223557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"ds_connection connected from Ok(127.0.0.1:63265)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084252283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084850026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"0","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084899309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"2","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.084964851Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"1","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085296018Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[0]R flush_numbers[0..12]: [45, 32, 38, 38, 40, 38, 44, 31, 45, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085323285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[0]R generation[0..12]: [21, 16, 18, 18, 20, 18, 20, 15, 21, 18, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085336823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085349441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[1]R flush_numbers[0..12]: [45, 32, 38, 38, 40, 38, 44, 31, 45, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.08536132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[1]R generation[0..12]: [21, 16, 18, 18, 20, 18, 20, 15, 21, 18, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085373588Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085385997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[2]R flush_numbers[0..12]: [45, 32, 38, 38, 40, 38, 44, 31, 45, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085397895Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[2]R generation[0..12]: [21, 16, 18, 18, 20, 18, 20, 15, 21, 18, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085409653Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085421872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.08543268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085443759Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085454238Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085464956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"":"downstairs","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085478125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085488243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"0","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085499122Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"1","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.08551Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"client":"2","":"downstairs","session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085521479Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085531828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"a26a7ac9-b368-4e9b-978c-ba50a0404c93 is now active with session: 1597d6aa-b777-4838-8fa4-ff2cb69cc377","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085543036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085552815Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"session_id":"1597d6aa-b777-4838-8fa4-ff2cb69cc377"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085563164Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Activated sub_volume a26a7ac9-b368-4e9b-978c-ba50a0404c93","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085572552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085581831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.085934766Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"task":"crutest"} 01/30 Write block 485 len 8192 data: 2 2 02/30 Write block 495 len 8192 data: 2 2 03/30 Read block 127 len 12288 04/30 Flush 05/30 Write block 514 len 20480 data: 4 4 4 7 5 06/30 Write block 71 len 8192 data: 2 2 07/30 Read block 129 len 32768 08/30 Write block 366 len 20480 data: 5 5 4 3 3 09/30 Write block 542 len 4096 data: 7 10/30 Flush 11/30 Write block 116 len 32768 data: 2 3 3 4 4 5 6 5 12/30 Flush 13/30 Write block 311 len 28672 data: 6 4 4 4 4 3 3 14/30 Read block 43 len 16384 15/30 Read block 517 len 36864 16/30 Read block 358 len 36864 17/30 Read block 532 len 20480 18/30 Read block 57 len 12288 19/30 Write block 266 len 4096 data: 3 20/30 Flush 21/30 Write block 319 len 32768 data: 2 3 4 4 4 3 4 4 22/30 Write block 306 len 24576 data: 4 5 4 5 5 7 23/30 Write block 399 len 20480 data: 2 2 2 3 3 24/30 Read block 326 len 8192 25/30 Read block 419 len 8192 26/30 Write block 340 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Read block 544 len 16384 28/30 Write block 416 len 28672 data: 3 3 3 3 4 4 3 29/30 Write block 391 len 32768 data: 6 5 5 6 6 5 6 3 30/30 Read block 148 len 12288 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 29 1 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:48:53.095071244Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:48:53.095281467Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1232,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 45 45 45 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 18 22 47 38 47 F F F <--- 4 080-099 20 20 20 40 40 40 F F F 5 100-119 22 18 22 48 38 48 F F F <--- 6 120-139 22 20 22 48 44 48 F F F <--- 7 140-159 15 15 15 31 31 31 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 20 20 20 41 41 41 F F F 11 220-239 20 20 20 41 41 41 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 22 18 22 49 38 49 F F F <--- 14 280-299 4 4 4 7 7 7 F F F 15 300-319 22 21 22 49 45 49 T F T <--- 16 320-339 21 21 21 45 45 45 T F T <--- 17 340-359 18 18 18 38 38 38 T F T <--- 18 360-379 22 21 22 47 45 47 F F F <--- 19 380-399 21 21 21 45 45 45 T F T <--- 20 400-419 18 18 18 38 38 38 T F T <--- 21 420-439 16 16 16 34 34 34 T F T <--- 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 22 10 22 46 18 46 F T F <--- 25 500-519 22 18 22 47 38 47 F F F <--- 26 520-539 19 19 19 39 39 39 F F F 27 540-559 22 19 22 47 39 47 F F F <--- 28 560-579 19 19 19 39 39 39 F F F 29 580-599 20 20 20 40 40 40 F F F Max gen: 22, Max flush: 49 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 45 45 45 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 20 20 20 40 40 40 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 22 22 22 48 48 48 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 20 20 20 41 41 41 F F F 11 220-239 20 20 20 41 41 41 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 22 22 22 49 49 49 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 23 23 23 50 50 50 F F F 18 360-379 22 22 22 47 47 47 F F F 19 380-399 23 23 23 50 50 50 F F F 20 400-419 23 23 23 50 50 50 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 16 16 16 34 34 34 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 22 22 22 46 46 46 F F F 25 500-519 22 22 22 47 47 47 F F F 26 520-539 19 19 19 39 39 39 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 19 19 19 39 39 39 F F F 29 580-599 20 20 20 40 40 40 F F F Max gen: 23, Max flush: 50 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.37790404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.386703731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387014341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387054486Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387067004Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387079693Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Crucible 312683b8-b237-48a7-b659-d24418ff472b has session id: e2a2baaa-71d0-4d89-bab0-ea75128187aa","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387404591Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Upstairs opts: Upstairs UUID: 312683b8-b237-48a7-b659-d24418ff472b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387441706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Crucible stats registered with UUID: 312683b8-b237-48a7-b659-d24418ff472b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387454044Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:07.387464543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.38763851Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387669866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387681995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387835555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387850023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"312683b8-b237-48a7-b659-d24418ff472b active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387889758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387899937Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"0","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387910415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"1","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387920894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"2","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387931233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387943791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.387962808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:48290)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.388037989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:55816)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.388064665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:62451)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.388076814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.388695574Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"0","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.388711182Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"1","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.38872285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"2","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389093592Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[0]R flush_numbers[0..12]: [45, 32, 38, 47, 40, 48, 48, 31, 45, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389120709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[0]R generation[0..12]: [21, 16, 18, 22, 20, 22, 22, 15, 21, 18, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389132497Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389144436Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[1]R flush_numbers[0..12]: [45, 32, 38, 47, 40, 48, 48, 31, 45, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389155604Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[1]R generation[0..12]: [21, 16, 18, 22, 20, 22, 22, 15, 21, 18, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389166863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389180701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[2]R flush_numbers[0..12]: [45, 32, 38, 47, 40, 48, 48, 31, 45, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389192009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[2]R generation[0..12]: [21, 16, 18, 22, 20, 22, 22, 15, 21, 18, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389203148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389214817Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389225025Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389235464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389245513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389255621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"":"downstairs","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.3892663Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389276139Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"0","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389286707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"1","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389296936Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"client":"2","":"downstairs","session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389307355Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389317063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"312683b8-b237-48a7-b659-d24418ff472b is now active with session: e2a2baaa-71d0-4d89-bab0-ea75128187aa","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389327712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389338281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"session_id":"e2a2baaa-71d0-4d89-bab0-ea75128187aa"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389348049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Activated sub_volume 312683b8-b237-48a7-b659-d24418ff472b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389356948Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389365927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.389666068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"task":"crutest"} 01/30 Flush 02/30 Write block 573 len 8192 data: 3 3 03/30 Read block 105 len 16384 04/30 Write block 224 len 40960 data: 7 8 7 7 5 3 3 3 3 3 05/30 Read block 321 len 16384 06/30 Write block 442 len 36864 data: 4 3 3 3 5 4 4 2 2 07/30 Read block 211 len 40960 08/30 Read block 274 len 20480 09/30 Write block 470 len 24576 data: 3 3 3 3 3 3 10/30 Read block 394 len 28672 11/30 Read block 173 len 40960 12/30 Read block 226 len 12288 13/30 Read block 499 len 24576 14/30 Flush 15/30 Flush 16/30 Write block 517 len 28672 data: 8 6 5 5 5 5 4 17/30 Read block 518 len 8192 18/30 Write block 442 len 16384 data: 5 4 4 4 19/30 Write block 489 len 40960 data: 2 2 2 3 3 3 3 3 2 2 20/30 Write block 0 len 20480 data: 3 3 3 2 2 21/30 Flush 22/30 Write block 219 len 20480 data: 4 6 5 7 7 23/30 Read block 83 len 28672 24/30 Write block 464 len 40960 data: 4 3 3 4 3 3 4 4 4 4 25/30 Write block 560 len 16384 data: 7 8 7 4 26/30 Read block 182 len 8192 27/30 Write block 224 len 20480 data: 8 9 8 8 6 28/30 Read block 435 len 36864 29/30 Write block 123 len 16384 data: 6 5 4 3 30/30 Read block 11 len 36864 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 29 2 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1000)) Some(JobId(1020)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:07.399811426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:07.400046006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1241,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 21 24 54 45 54 F F F <--- 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 20 20 20 40 40 40 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 22 22 22 48 48 48 T F T <--- 7 140-159 15 15 15 31 31 31 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 20 20 20 41 41 41 T F T <--- 11 220-239 24 20 24 52 41 52 T F T <--- 12 240-259 17 17 17 37 37 37 F F F 13 260-279 22 22 22 49 49 49 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 23 23 23 50 50 50 F F F 18 360-379 22 22 22 47 47 47 F F F 19 380-399 23 23 23 50 50 50 F F F 20 400-419 23 23 23 50 50 50 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 24 16 24 54 34 54 F F F <--- 23 460-479 24 6 24 52 12 52 T F T <--- 24 480-499 24 22 24 54 46 54 F F F <--- 25 500-519 24 22 24 54 47 54 F F F <--- 26 520-539 24 19 24 54 39 54 F F F <--- 27 540-559 22 22 22 47 47 47 F F F 28 560-579 24 19 24 52 39 52 T T T <--- 29 580-599 20 20 20 40 40 40 F F F Max gen: 24, Max flush: 54 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 20 20 20 40 40 40 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 25 25 25 55 55 55 F F F 11 220-239 25 25 25 55 55 55 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 22 22 22 49 49 49 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 23 23 23 50 50 50 F F F 18 360-379 22 22 22 47 47 47 F F F 19 380-399 23 23 23 50 50 50 F F F 20 400-419 23 23 23 50 50 50 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 25 25 25 55 55 55 F F F 24 480-499 24 24 24 54 54 54 F F F 25 500-519 24 24 24 54 54 54 F F F 26 520-539 24 24 24 54 54 54 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 25 25 25 55 55 55 F F F 29 580-599 20 20 20 40 40 40 F F F Max gen: 25, Max flush: 55 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.666921515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676076071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676430225Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676468031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676478059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676491348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Crucible 9e3371c6-45fb-4c63-9ebd-4c6c328e1a72 has session id: fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676930321Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Upstairs opts: Upstairs UUID: 9e3371c6-45fb-4c63-9ebd-4c6c328e1a72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676965496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Crucible stats registered with UUID: 9e3371c6-45fb-4c63-9ebd-4c6c328e1a72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.676978754Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:21.676989633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677113537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"2","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677133684Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"1","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677145243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"0","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677333269Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677348297Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"9e3371c6-45fb-4c63-9ebd-4c6c328e1a72 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677379303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677389921Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"0","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"1","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677412268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"2","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677423397Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"2","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677434795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"0","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677446334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"1","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40112)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.67755664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"1","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47329)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.67763404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"2","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42754)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.677647528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"io task","client":"0","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678334529Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"2","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678380243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"1","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.67840283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"0","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678781791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[0]R flush_numbers[0..12]: [54, 32, 38, 47, 40, 48, 55, 31, 45, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678798779Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[0]R generation[0..12]: [24, 16, 18, 22, 20, 22, 25, 15, 21, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678810978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678823586Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[1]R flush_numbers[0..12]: [54, 32, 38, 47, 40, 48, 55, 31, 45, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678835394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[1]R generation[0..12]: [24, 16, 18, 22, 20, 22, 25, 15, 21, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678847033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678859301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[2]R flush_numbers[0..12]: [54, 32, 38, 47, 40, 48, 55, 31, 45, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.67887106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[2]R generation[0..12]: [24, 16, 18, 22, 20, 22, 25, 15, 21, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678883028Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678895267Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678905905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678916934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678929312Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678940011Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"":"downstairs","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678950949Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678960998Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"0","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678972067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"1","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678983155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"client":"2","":"downstairs","session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.678994344Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.679004882Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"9e3371c6-45fb-4c63-9ebd-4c6c328e1a72 is now active with session: fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.679016361Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.67902634Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"session_id":"fb9c2f17-b49f-4ef1-8e67-3d73f1a00ba5"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.679036638Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Activated sub_volume 9e3371c6-45fb-4c63-9ebd-4c6c328e1a72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.679045807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.679055036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.679429567Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"task":"crutest"} 01/30 Flush 02/30 Write block 343 len 20480 data: 3 3 3 3 3 03/30 Write block 503 len 4096 data: 2 04/30 Read block 289 len 4096 05/30 Read block 365 len 28672 06/30 Read block 86 len 20480 07/30 Read block 13 len 28672 08/30 Write block 468 len 24576 data: 4 4 5 5 5 5 09/30 Flush 10/30 Write block 383 len 40960 data: 3 2 2 3 3 4 6 6 7 6 11/30 Read block 66 len 36864 12/30 Read block 157 len 24576 13/30 Read block 330 len 36864 14/30 Read block 417 len 8192 15/30 Read block 259 len 20480 16/30 Read block 48 len 40960 17/30 Flush 18/30 Write block 575 len 24576 data: 3 3 4 4 4 4 19/30 Write block 267 len 40960 data: 4 4 4 4 3 3 4 4 4 3 20/30 Read block 281 len 16384 21/30 Read block 421 len 20480 22/30 Write block 407 len 8192 data: 3 3 23/30 Read block 575 len 24576 24/30 Write block 442 len 24576 data: 6 5 5 5 6 5 25/30 Write block 92 len 12288 data: 3 3 5 26/30 Write block 162 len 8192 data: 5 5 27/30 Read block 86 len 24576 28/30 Read block 199 len 20480 29/30 Read block 77 len 40960 30/30 Write block 498 len 12288 data: 3 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:21.689370732Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:21.689581294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1250,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 20 20 20 40 40 40 T F T <--- 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 21 21 21 45 45 45 T F T <--- 9 180-199 18 18 18 38 38 38 F F F 10 200-219 25 25 25 55 55 55 F F F 11 220-239 25 25 25 55 55 55 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 22 22 22 49 49 49 T F T <--- 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 26 23 26 57 50 57 F F F <--- 18 360-379 22 22 22 47 47 47 F F F 19 380-399 26 23 26 58 50 58 F F F <--- 20 400-419 23 23 23 50 50 50 T F T <--- 21 420-439 23 23 23 50 50 50 F F F 22 440-459 24 24 24 54 54 54 T F T <--- 23 460-479 26 25 26 57 55 57 F F F <--- 24 480-499 24 24 24 54 54 54 T F T <--- 25 500-519 26 24 26 57 54 57 T F T <--- 26 520-539 24 24 24 54 54 54 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 25 25 25 55 55 55 T F T <--- 29 580-599 20 20 20 40 40 40 T F T <--- Max gen: 26, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 16 16 16 32 32 32 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 27 27 27 59 59 59 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 15 15 15 31 31 31 F F F 8 160-179 27 27 27 59 59 59 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 25 25 25 55 55 55 F F F 11 220-239 25 25 25 55 55 55 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 27 27 27 59 59 59 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 26 26 26 57 57 57 F F F 18 360-379 22 22 22 47 47 47 F F F 19 380-399 26 26 26 58 58 58 F F F 20 400-419 27 27 27 59 59 59 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 26 26 26 57 57 57 F F F 24 480-499 27 27 27 59 59 59 F F F 25 500-519 27 27 27 59 59 59 F F F 26 520-539 24 24 24 54 54 54 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 27 27 27 59 59 59 F F F 29 580-599 27 27 27 59 59 59 F F F Max gen: 27, Max flush: 59 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.96931881Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.977865534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978183033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978214389Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978225608Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978237536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Crucible bfd4329b-0d16-4548-8cd6-3bc8730f1c27 has session id: d79d2831-a1b6-4305-8e0e-3ac6e020bd58","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978642954Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Upstairs opts: Upstairs UUID: bfd4329b-0d16-4548-8cd6-3bc8730f1c27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.97867362Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Crucible stats registered with UUID: bfd4329b-0d16-4548-8cd6-3bc8730f1c27","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978685148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:35.978694907Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978816361Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978833469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.978843918Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979007777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979017845Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"bfd4329b-0d16-4548-8cd6-3bc8730f1c27 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979035353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979044422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"0","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979054451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"1","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979064379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"2","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979076018Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979086237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979117642Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"ds_connection connected from Ok(127.0.0.1:47181)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.97913892Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"ds_connection connected from Ok(127.0.0.1:60309)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979169426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"ds_connection connected from Ok(127.0.0.1:54897)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.97921219Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979734833Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"1","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979760239Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"0","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.979915399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"2","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980156048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[0]R flush_numbers[0..12]: [54, 32, 38, 47, 59, 48, 55, 31, 59, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980176556Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[0]R generation[0..12]: [24, 16, 18, 22, 27, 22, 25, 15, 27, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980187774Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980199393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[1]R flush_numbers[0..12]: [54, 32, 38, 47, 59, 48, 55, 31, 59, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980210341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[1]R generation[0..12]: [24, 16, 18, 22, 27, 22, 25, 15, 27, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.98022303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980234558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[2]R flush_numbers[0..12]: [54, 32, 38, 47, 59, 48, 55, 31, 59, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980245307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[2]R generation[0..12]: [24, 16, 18, 22, 27, 22, 25, 15, 27, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980256095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980267304Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980277053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980287141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.98029687Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980306629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"":"downstairs","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980316997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980326076Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"0","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980336495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"1","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980346634Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"client":"2","":"downstairs","session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980357142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980366701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"bfd4329b-0d16-4548-8cd6-3bc8730f1c27 is now active with session: d79d2831-a1b6-4305-8e0e-3ac6e020bd58","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980378699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980387718Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"session_id":"d79d2831-a1b6-4305-8e0e-3ac6e020bd58"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980397207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Activated sub_volume bfd4329b-0d16-4548-8cd6-3bc8730f1c27","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980405676Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980414285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.980764889Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"task":"crutest"} 01/30 Flush 02/30 Write block 508 len 24576 data: 3 3 3 6 6 5 03/30 Write block 159 len 8192 data: 4 4 04/30 Read block 242 len 16384 05/30 Read block 511 len 28672 06/30 Read block 537 len 32768 07/30 Read block 107 len 12288 08/30 Write block 270 len 8192 data: 5 4 09/30 Flush 10/30 Write block 29 len 4096 data: 3 11/30 Read block 510 len 16384 12/30 Read block 45 len 24576 13/30 Read block 327 len 4096 14/30 Read block 223 len 12288 15/30 Read block 475 len 20480 16/30 Read block 391 len 12288 17/30 Write block 464 len 40960 data: 5 4 4 5 5 5 6 6 6 6 18/30 Read block 137 len 8192 19/30 Write block 85 len 16384 data: 3 3 3 3 20/30 Read block 283 len 32768 21/30 Read block 33 len 4096 22/30 Write block 535 len 20480 data: 3 4 5 8 7 23/30 Flush 24/30 Flush 25/30 Write block 492 len 20480 data: 4 4 4 4 4 26/30 Read block 504 len 32768 27/30 Write block 490 len 24576 data: 3 3 5 5 5 5 28/30 Write block 598 len 8192 data: 3 2 29/30 Read block 242 len 8192 30/30 Write block 398 len 28672 data: 4 3 3 3 4 4 2 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:35.989300525Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:35.989487891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1259,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 28 28 16 62 62 32 F F F <--- 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 28 28 27 62 62 59 F F F <--- 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 28 28 15 61 61 31 F F F <--- 8 160-179 28 28 27 61 61 59 F F F <--- 9 180-199 18 18 18 38 38 38 F F F 10 200-219 25 25 25 55 55 55 F F F 11 220-239 25 25 25 55 55 55 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 28 28 27 61 61 59 F F F <--- 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 26 26 26 57 57 57 F F F 18 360-379 22 22 22 47 47 47 F F F 19 380-399 26 26 26 58 58 58 T T F <--- 20 400-419 27 27 27 59 59 59 T T F <--- 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 28 28 26 62 62 57 F F F <--- 24 480-499 27 27 27 59 59 59 T T F <--- 25 500-519 28 28 27 61 61 59 F F F <--- 26 520-539 28 28 24 62 62 54 F F F <--- 27 540-559 22 22 22 47 47 47 F F F 28 560-579 27 27 27 59 59 59 F F F 29 580-599 27 27 27 59 59 59 T T F <--- Max gen: 28, Max flush: 62 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 28 28 28 61 61 61 F F F 8 160-179 28 28 28 61 61 61 F F F 9 180-199 18 18 18 38 38 38 F F F 10 200-219 25 25 25 55 55 55 F F F 11 220-239 25 25 25 55 55 55 F F F 12 240-259 17 17 17 37 37 37 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 23 23 23 50 50 50 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 26 26 26 57 57 57 F F F 18 360-379 22 22 22 47 47 47 F F F 19 380-399 29 29 29 63 63 63 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 28 28 28 62 62 62 F F F 24 480-499 29 29 29 63 63 63 F F F 25 500-519 28 28 28 61 61 61 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 27 27 27 59 59 59 F F F 29 580-599 29 29 29 63 63 63 F F F Max gen: 29, Max flush: 63 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.258124353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.266674737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.266980568Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267012473Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267021822Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267034001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Crucible 8f1b94f3-8f31-48c3-b535-31854ba3dfef has session id: 82d0b7d5-33e9-42ee-9fe3-097bb1659bcd","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267421521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Upstairs opts: Upstairs UUID: 8f1b94f3-8f31-48c3-b535-31854ba3dfef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267452447Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Crucible stats registered with UUID: 8f1b94f3-8f31-48c3-b535-31854ba3dfef","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267464055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:50.267473934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267604157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267636023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267647571Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267791443Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267805581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"8f1b94f3-8f31-48c3-b535-31854ba3dfef active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267843736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267853875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"0","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267864403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"1","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267874872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"2","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267885161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267896069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.267906788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60865)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.268021413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"ds_connection connected from Ok(127.0.0.1:34172)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.268033941Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"ds_connection connected from Ok(127.0.0.1:54729)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.268080725Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.268732301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"0","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.268748609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"1","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.268860015Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"2","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269162875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[0]R flush_numbers[0..12]: [54, 62, 38, 47, 62, 48, 55, 61, 61, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269199841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[0]R generation[0..12]: [24, 28, 18, 22, 28, 22, 25, 28, 28, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269211579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269223488Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[1]R flush_numbers[0..12]: [54, 62, 38, 47, 62, 48, 55, 61, 61, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269234846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[1]R generation[0..12]: [24, 28, 18, 22, 28, 22, 25, 28, 28, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269246175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269258063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[2]R flush_numbers[0..12]: [54, 62, 38, 47, 62, 48, 55, 61, 61, 38, 55, 55]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269269292Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[2]R generation[0..12]: [24, 28, 18, 22, 28, 22, 25, 28, 28, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.26928048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269292579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269302957Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269313506Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269325364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269335363Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"":"downstairs","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269345972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.26935541Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"0","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269365999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"1","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269376518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"client":"2","":"downstairs","session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269387236Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269397095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"8f1b94f3-8f31-48c3-b535-31854ba3dfef is now active with session: 82d0b7d5-33e9-42ee-9fe3-097bb1659bcd","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269407894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269417392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"session_id":"82d0b7d5-33e9-42ee-9fe3-097bb1659bcd"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269427241Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Activated sub_volume 8f1b94f3-8f31-48c3-b535-31854ba3dfef","v":0,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "name":"crucible","level":30,"time":"2026-05-14T20:49:50.269442249Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269451338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.269706475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"task":"crutest"} 01/30 Write block 216 len 8192 data: 2 2 02/30 Read block 468 len 4096 03/30 Read block 149 len 20480 04/30 Read block 521 len 12288 05/30 Write block 354 len 28672 data: 3 3 3 3 4 4 4 06/30 Read block 387 len 8192 07/30 Read block 337 len 4096 08/30 Read block 396 len 28672 09/30 Read block 378 len 4096 10/30 Read block 445 len 28672 11/30 Read block 473 len 36864 12/30 Read block 326 len 4096 13/30 Read block 56 len 20480 14/30 Read block 502 len 32768 15/30 Write block 167 len 8192 data: 2 2 16/30 Read block 473 len 20480 17/30 Flush 18/30 Write block 286 len 40960 data: 3 3 3 3 3 3 3 3 3 3 19/30 Write block 60 len 8192 data: 3 4 20/30 Read block 498 len 28672 21/30 Write block 490 len 8192 data: 4 4 22/30 Read block 502 len 40960 23/30 Read block 352 len 32768 24/30 Read block 344 len 20480 25/30 Write block 180 len 20480 data: 2 2 3 4 4 26/30 Write block 304 len 16384 data: 3 3 5 6 27/30 Read block 10 len 36864 28/30 Read block 45 len 16384 29/30 Write block 238 len 28672 data: 2 2 2 4 4 4 5 30/30 Read block 296 len 32768 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:49:50.278653798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:49:50.278851822Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1268,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 22 22 22 47 47 47 T F T <--- 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 28 28 28 61 61 61 F F F 8 160-179 30 28 30 64 61 64 F F F <--- 9 180-199 18 18 18 38 38 38 T F T <--- 10 200-219 30 25 30 64 55 64 F F F <--- 11 220-239 25 25 25 55 55 55 T F T <--- 12 240-259 17 17 17 37 37 37 T F T <--- 13 260-279 28 28 28 61 61 61 F F F 14 280-299 4 4 4 7 7 7 T F T <--- 15 300-319 23 23 23 50 50 50 T F T <--- 16 320-339 23 23 23 50 50 50 F F F 17 340-359 30 26 30 64 57 64 F F F <--- 18 360-379 30 22 30 64 47 64 F F F <--- 19 380-399 29 29 29 63 63 63 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 28 28 28 62 62 62 F F F 24 480-499 29 29 29 63 63 63 T F T <--- 25 500-519 28 28 28 61 61 61 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 27 27 27 59 59 59 F F F 29 580-599 29 29 29 63 63 63 F F F Max gen: 30, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 18 18 18 38 38 38 F F F 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 28 28 28 61 61 61 F F F 8 160-179 30 30 30 64 64 64 F F F 9 180-199 31 31 31 65 65 65 F F F 10 200-219 30 30 30 64 64 64 F F F 11 220-239 31 31 31 65 65 65 F F F 12 240-259 31 31 31 65 65 65 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 31 31 31 65 65 65 F F F 16 320-339 23 23 23 50 50 50 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 30 30 30 64 64 64 F F F 19 380-399 29 29 29 63 63 63 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 28 28 28 62 62 62 F F F 24 480-499 31 31 31 65 65 65 F F F 25 500-519 28 28 28 61 61 61 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 27 27 27 59 59 59 F F F 29 580-599 29 29 29 63 63 63 F F F Max gen: 31, Max flush: 65 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.552071681Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.560990408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.561370748Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.561408744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.561421882Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.56143464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Crucible 2590b868-fed3-48fa-bf57-8d5ba8488ed0 has session id: 9a377b2e-0323-4830-86a6-3d55d0cc1612","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.56181946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Upstairs opts: Upstairs UUID: 2590b868-fed3-48fa-bf57-8d5ba8488ed0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.561837558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Crucible stats registered with UUID: 2590b868-fed3-48fa-bf57-8d5ba8488ed0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.561851716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:04.561862005Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562012475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562031713Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562042502Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.56220937Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562219869Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"2590b868-fed3-48fa-bf57-8d5ba8488ed0 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562245425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562255674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"0","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562266353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"1","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562278811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"2","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.56228943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562300378Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562347682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"ds_connection connected from Ok(127.0.0.1:62565)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562359671Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"ds_connection connected from Ok(127.0.0.1:62076)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562403445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"ds_connection connected from Ok(127.0.0.1:51284)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.562457868Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563141869Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"0","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563156697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"1","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563205341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"2","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563516421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[0]R flush_numbers[0..12]: [54, 62, 38, 65, 62, 48, 55, 61, 64, 65, 64, 65]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563538398Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[0]R generation[0..12]: [24, 28, 18, 31, 28, 22, 25, 28, 30, 31, 30, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563550217Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563562515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[1]R flush_numbers[0..12]: [54, 62, 38, 65, 62, 48, 55, 61, 64, 65, 64, 65]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563577003Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[1]R generation[0..12]: [24, 28, 18, 31, 28, 22, 25, 28, 30, 31, 30, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563588582Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563601Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[2]R flush_numbers[0..12]: [54, 62, 38, 65, 62, 48, 55, 61, 64, 65, 64, 65]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563613008Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[2]R generation[0..12]: [24, 28, 18, 31, 28, 22, 25, 28, 30, 31, 30, 31]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563624717Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563637055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563647604Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563658493Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563668911Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.56367946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"":"downstairs","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563690688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563700537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"0","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563711566Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"1","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563722584Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"client":"2","":"downstairs","session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563733763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563744012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"2590b868-fed3-48fa-bf57-8d5ba8488ed0 is now active with session: 9a377b2e-0323-4830-86a6-3d55d0cc1612","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.56375619Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563765899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"session_id":"9a377b2e-0323-4830-86a6-3d55d0cc1612"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563776117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Activated sub_volume 2590b868-fed3-48fa-bf57-8d5ba8488ed0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563785086Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.563794135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.564165197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"task":"crutest"} 01/30 Write block 562 len 4096 data: 8 02/30 Read block 450 len 4096 03/30 Flush 04/30 Write block 326 len 24576 data: 5 5 6 5 6 7 05/30 Read block 563 len 4096 06/30 Read block 542 len 32768 07/30 Write block 304 len 24576 data: 4 4 6 7 5 6 08/30 Read block 275 len 32768 09/30 Write block 313 len 12288 data: 5 5 5 10/30 Write block 48 len 8192 data: 4 5 11/30 Flush 12/30 Write block 369 len 40960 data: 4 4 2 2 2 3 3 4 4 3 13/30 Write block 381 len 8192 data: 5 5 14/30 Read block 233 len 24576 15/30 Read block 399 len 8192 16/30 Read block 531 len 12288 17/30 Read block 74 len 12288 18/30 Write block 465 len 20480 data: 5 5 6 6 6 19/30 Write block 387 len 16384 data: 4 5 7 7 20/30 Write block 207 len 24576 data: 5 5 5 4 2 2 21/30 Read block 206 len 24576 22/30 Write block 365 len 32768 data: 5 6 6 5 5 5 3 3 23/30 Write block 506 len 24576 data: 3 3 4 4 4 7 24/30 Read block 170 len 36864 25/30 Write block 148 len 16384 data: 2 2 2 2 26/30 Read block 269 len 36864 27/30 Write block 213 len 28672 data: 3 2 2 3 3 4 5 28/30 Write block 4 len 20480 data: 3 4 4 4 4 29/30 Write block 218 len 16384 data: 5 6 7 6 30/30 Read block 450 len 36864 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 1 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:04.573280428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:04.573489251Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 54 54 54 T T F <--- 1 020-039 28 28 28 62 62 62 F F F 2 040-059 32 32 18 67 67 38 F F F <--- 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 28 28 28 61 61 61 T T F <--- 8 160-179 30 30 30 64 64 64 F F F 9 180-199 31 31 31 65 65 65 F F F 10 200-219 30 30 30 64 64 64 T T F <--- 11 220-239 31 31 31 65 65 65 T T F <--- 12 240-259 31 31 31 65 65 65 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 32 32 31 67 67 65 F F F <--- 16 320-339 32 32 23 67 67 50 F F F <--- 17 340-359 30 30 30 64 64 64 F F F 18 360-379 30 30 30 64 64 64 T T F <--- 19 380-399 29 29 29 63 63 63 T T F <--- 20 400-419 29 29 29 63 63 63 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 28 28 28 62 62 62 T T F <--- 24 480-499 31 31 31 65 65 65 F F F 25 500-519 28 28 28 61 61 61 T T F <--- 26 520-539 28 28 28 62 62 62 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 32 32 27 66 66 59 F F F <--- 29 580-599 29 29 29 63 63 63 F F F Max gen: 32, Max flush: 67 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 68 68 68 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 32 32 32 67 67 67 F F F 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 30 30 30 64 64 64 F F F 9 180-199 31 31 31 65 65 65 F F F 10 200-219 33 33 33 68 68 68 F F F 11 220-239 33 33 33 68 68 68 F F F 12 240-259 31 31 31 65 65 65 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 32 32 32 67 67 67 F F F 16 320-339 32 32 32 67 67 67 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 33 33 33 68 68 68 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 23 23 23 50 50 50 F F F 22 440-459 27 27 27 59 59 59 F F F 23 460-479 33 33 33 68 68 68 F F F 24 480-499 31 31 31 65 65 65 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 22 22 22 47 47 47 F F F 28 560-579 32 32 32 66 66 66 F F F 29 580-599 29 29 29 63 63 63 F F F Max gen: 33, Max flush: 68 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.849658738Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.858569935Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.858887814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.85892193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.858931759Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.858944827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Crucible be231544-7426-4f6d-893f-888b0fbf12ee has session id: e7ad68fe-a79f-463f-8376-c2ac751d95d0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859358973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Upstairs opts: Upstairs UUID: be231544-7426-4f6d-893f-888b0fbf12ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859378681Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Crucible stats registered with UUID: be231544-7426-4f6d-893f-888b0fbf12ee","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859395569Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:18.859405797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859548669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859566957Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859578275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859766821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859777649Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"be231544-7426-4f6d-893f-888b0fbf12ee active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859813375Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859824053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"0","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859835132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"1","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.85984604Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"2","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859857039Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859868478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859879566Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39529)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859936219Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38643)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859948557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61259)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.859960146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.860767981Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"1","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.860789828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"2","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.860821614Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"0","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861188257Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[0]R flush_numbers[0..12]: [68, 62, 67, 65, 62, 48, 55, 68, 64, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861211304Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[0]R generation[0..12]: [33, 28, 32, 31, 28, 22, 25, 33, 30, 31, 33, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861223272Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861235661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[1]R flush_numbers[0..12]: [68, 62, 67, 65, 62, 48, 55, 68, 64, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861247429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[1]R generation[0..12]: [33, 28, 32, 31, 28, 22, 25, 33, 30, 31, 33, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861258878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861271216Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[2]R flush_numbers[0..12]: [68, 62, 67, 65, 62, 48, 55, 68, 64, 65, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861282895Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[2]R generation[0..12]: [33, 28, 32, 31, 28, 22, 25, 33, 30, 31, 33, 33]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861294513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861306651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.86131938Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861330598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861341337Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861351856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"":"downstairs","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861362974Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861372893Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"0","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861384021Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"1","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.86139507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"client":"2","":"downstairs","session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861406209Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861416557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"be231544-7426-4f6d-893f-888b0fbf12ee is now active with session: e7ad68fe-a79f-463f-8376-c2ac751d95d0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861427726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861437575Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"session_id":"e7ad68fe-a79f-463f-8376-c2ac751d95d0"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861447843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Activated sub_volume be231544-7426-4f6d-893f-888b0fbf12ee","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861456922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861466191Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.861843972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"task":"crutest"} 01/30 Write block 433 len 40960 data: 4 4 3 3 5 5 4 5 5 7 02/30 Read block 310 len 20480 03/30 Write block 469 len 32768 data: 7 7 7 7 7 4 4 2 04/30 Read block 4 len 16384 05/30 Write block 550 len 20480 data: 4 3 3 4 5 06/30 Write block 201 len 20480 data: 3 4 4 4 4 07/30 Read block 296 len 24576 08/30 Read block 136 len 4096 09/30 Read block 502 len 28672 10/30 Read block 43 len 20480 11/30 Read block 425 len 4096 12/30 Flush 13/30 Write block 217 len 36864 data: 4 6 7 8 7 8 8 9 10 14/30 Read block 247 len 4096 15/30 Read block 219 len 16384 16/30 Read block 131 len 8192 17/30 Flush 18/30 Write block 583 len 32768 data: 2 2 3 2 2 2 4 5 19/30 Read block 115 len 28672 20/30 Read block 556 len 40960 21/30 Read block 17 len 8192 22/30 Write block 541 len 8192 data: 7 8 23/30 Read block 168 len 16384 24/30 Read block 363 len 32768 25/30 Read block 544 len 40960 26/30 Read block 590 len 24576 27/30 Read block 503 len 24576 28/30 Write block 454 len 28672 data: 2 2 2 3 4 5 5 29/30 Read block 167 len 8192 30/30 Read block 163 len 12288 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:18.871903231Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:18.872109014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1286,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 68 68 68 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 32 32 32 67 67 67 F F F 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 30 30 30 64 64 64 F F F 9 180-199 31 31 31 65 65 65 F F F 10 200-219 34 33 34 70 68 70 F F F <--- 11 220-239 34 33 34 70 68 70 F F F <--- 12 240-259 31 31 31 65 65 65 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 32 32 32 67 67 67 F F F 16 320-339 32 32 32 67 67 67 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 33 33 33 68 68 68 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 23 34 69 50 69 F T F <--- 22 440-459 34 27 34 69 59 69 T T T <--- 23 460-479 34 33 34 69 68 69 T F T <--- 24 480-499 31 31 31 65 65 65 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 34 22 34 69 47 69 T F T <--- 28 560-579 32 32 32 66 66 66 F F F 29 580-599 29 29 29 63 63 63 T F T <--- Max gen: 34, Max flush: 70 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 68 68 68 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 32 32 32 67 67 67 F F F 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 30 30 30 64 64 64 F F F 9 180-199 31 31 31 65 65 65 F F F 10 200-219 34 34 34 70 70 70 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 31 31 31 65 65 65 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 32 32 32 67 67 67 F F F 16 320-339 32 32 32 67 67 67 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 33 33 33 68 68 68 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 35 35 35 71 71 71 F F F 23 460-479 35 35 35 71 71 71 F F F 24 480-499 31 31 31 65 65 65 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 35 35 35 71 71 71 F F F 28 560-579 32 32 32 66 66 66 F F F 29 580-599 35 35 35 71 71 71 F F F Max gen: 35, Max flush: 71 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.139610953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:33.147463527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.147525049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.155345628Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.155645169Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.155677395Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.155686714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.155699302Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Crucible a066d53c-7317-4b65-9405-5471b7841d89 has session id: e0d59865-27d0-4287-a48e-35b7d64a5e48","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156131246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Upstairs opts: Upstairs UUID: a066d53c-7317-4b65-9405-5471b7841d89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156162522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Crucible stats registered with UUID: a066d53c-7317-4b65-9405-5471b7841d89","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156174341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:33.156184439Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156315472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.15633375Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156345148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156513117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156531544Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"a066d53c-7317-4b65-9405-5471b7841d89 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.15656231Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156572589Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"0","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156583368Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"1","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156593876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"2","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156604435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156615484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156626452Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:63133)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156694333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:60521)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.1567225Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:52341)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.156735678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.157505748Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"1","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.157518807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"0","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.157566151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"2","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.157952191Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[0]R flush_numbers[0..12]: [68, 62, 67, 65, 62, 48, 55, 68, 64, 65, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.157976977Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[0]R generation[0..12]: [33, 28, 32, 31, 28, 22, 25, 33, 30, 31, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.157992275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158015952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[1]R flush_numbers[0..12]: [68, 62, 67, 65, 62, 48, 55, 68, 64, 65, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158027571Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[1]R generation[0..12]: [33, 28, 32, 31, 28, 22, 25, 33, 30, 31, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158039079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158051338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[2]R flush_numbers[0..12]: [68, 62, 67, 65, 62, 48, 55, 68, 64, 65, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158062926Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[2]R generation[0..12]: [33, 28, 32, 31, 28, 22, 25, 33, 30, 31, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158074385Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158086433Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158102691Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.15811345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158124058Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158134487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"":"downstairs","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158145286Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158155124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"0","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158167173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"1","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158177831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"client":"2","":"downstairs","session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.15818879Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158198939Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"a066d53c-7317-4b65-9405-5471b7841d89 is now active with session: e0d59865-27d0-4287-a48e-35b7d64a5e48","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158210137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158219976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"session_id":"e0d59865-27d0-4287-a48e-35b7d64a5e48"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158229935Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Activated sub_volume a066d53c-7317-4b65-9405-5471b7841d89","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158239104Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158248282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.158584999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} 01/30 Write block 1 len 20480 data: 4 4 3 4 5 02/30 Read block 336 len 24576 03/30 Write block 42 len 32768 data: 3 3 4 6 5 5 5 6 04/30 Read block 140 len 8192 05/30 Write block 468 len 20480 data: 7 8 8 8 8 06/30 Write block 579 len 28672 data: 5 5 2 2 3 3 4 07/30 Write block 7 len 36864 data: 5 5 4 5 5 4 3 3 4 08/30 Read block 430 len 40960 09/30 Read block 440 len 20480 10/30 Write block 382 len 40960 data: 6 4 3 3 4 5 6 8 8 8 11/30 Read block 534 len 12288 12/30 Read block 206 len 40960 13/30 Write block 543 len 12288 data: 4 2 3 14/30 Write block 246 len 12288 data: 5 4 3 15/30 Read block 48 len 40960 16/30 Read block 60 len 20480 17/30 Read block 113 len 24576 18/30 Write block 176 len 12288 data: 4 3 3 19/30 Read block 77 len 20480 20/30 Read block 179 len 24576 21/30 Read block 35 len 36864 22/30 Write block 511 len 16384 data: 8 7 6 5 23/30 Write block 132 len 20480 data: 3 3 3 3 3 24/30 Read block 117 len 12288 25/30 Write block 327 len 12288 data: 6 7 6 26/30 Read block 559 len 4096 27/30 Write block 171 len 4096 data: 3 28/30 Read block 280 len 8192 29/30 Read block 286 len 24576 30/30 Write block 188 len 20480 data: 2 2 2 2 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:33.168348056Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:33.168569077Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1295,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 68 68 68 T T T 1 020-039 28 28 28 62 62 62 F F F 2 040-059 32 32 32 67 67 67 T T T 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 25 25 25 55 55 55 F T T <--- 7 140-159 33 33 33 68 68 68 F F F 8 160-179 30 30 30 64 64 64 F T T <--- 9 180-199 31 31 31 65 65 65 F T T <--- 10 200-219 34 34 34 70 70 70 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 31 31 31 65 65 65 F T T <--- 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 32 32 32 67 67 67 F F F 16 320-339 32 32 32 67 67 67 F T T <--- 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 33 33 33 68 68 68 F T T <--- 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 35 35 35 71 71 71 F F F 23 460-479 35 35 35 71 71 71 F T T <--- 24 480-499 31 31 31 65 65 65 F F F 25 500-519 33 33 33 68 68 68 F T T <--- 26 520-539 28 28 28 62 62 62 F F F 27 540-559 35 35 35 71 71 71 F T T <--- 28 560-579 32 32 32 66 66 66 F T T <--- 29 580-599 35 35 35 71 71 71 F T T <--- Max gen: 35, Max flush: 71 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 9a10,11 > 5, > 5, 10a13,14 > 5, > 5, 12a17 > 3, 14d18 < 4, 16,17d19 < 2, < 2, 19,20d20 < 3, < 3, diff found after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 72 72 72 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 36 36 36 72 72 72 F F F 3 060-079 31 31 31 65 65 65 F F F 4 080-099 28 28 28 62 62 62 F F F 5 100-119 22 22 22 48 48 48 F F F 6 120-139 36 36 36 72 72 72 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 36 36 36 72 72 72 F F F 9 180-199 36 36 36 72 72 72 F F F 10 200-219 34 34 34 70 70 70 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 36 36 36 72 72 72 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 31 31 31 65 65 65 F F F 15 300-319 32 32 32 67 67 67 F F F 16 320-339 36 36 36 72 72 72 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 35 35 35 71 71 71 F F F 23 460-479 36 36 36 72 72 72 F F F 24 480-499 31 31 31 65 65 65 F F F 25 500-519 36 36 36 72 72 72 F F F 26 520-539 28 28 28 62 62 62 F F F 27 540-559 36 36 36 72 72 72 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 36 36 36 72 72 72 F F F Max gen: 36, Max flush: 72 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.455192333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.463778982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464074274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.46410671Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464116099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464129827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Crucible 2df9a117-0925-4254-bd50-3c3cba4b9813 has session id: e25eec74-eb61-43db-8633-263f865e863b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.46456744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Upstairs opts: Upstairs UUID: 2df9a117-0925-4254-bd50-3c3cba4b9813, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464599546Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Crucible stats registered with UUID: 2df9a117-0925-4254-bd50-3c3cba4b9813","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464611655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:47.464621423Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464755466Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464776153Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464787062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464963539Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.464981397Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"2df9a117-0925-4254-bd50-3c3cba4b9813 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465010143Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465019752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"0","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.46503021Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"1","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465040739Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"2","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465051218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465061976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465103991Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57098)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465129788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49212)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465141916Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33085)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465245303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465821718Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"1","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465864532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"0","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.465969029Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"2","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466230855Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[0]R flush_numbers[0..12]: [72, 62, 72, 65, 62, 48, 72, 68, 72, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466257702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[0]R generation[0..12]: [36, 28, 36, 31, 28, 22, 36, 33, 36, 36, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.4662694Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466281589Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[1]R flush_numbers[0..12]: [72, 62, 72, 65, 62, 48, 72, 68, 72, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466292807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[1]R generation[0..12]: [36, 28, 36, 31, 28, 22, 36, 33, 36, 36, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466303866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466315584Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[2]R flush_numbers[0..12]: [72, 62, 72, 65, 62, 48, 72, 68, 72, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466328782Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[2]R generation[0..12]: [36, 28, 36, 31, 28, 22, 36, 33, 36, 36, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466339841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466351599Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Max found gen is 37","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466361818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Generation requested: 38 >= found:37","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466372367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466382435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466392454Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"":"downstairs","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466402903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466412202Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"0","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.46642254Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"1","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466433199Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"client":"2","":"downstairs","session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466443888Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466453686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"2df9a117-0925-4254-bd50-3c3cba4b9813 is now active with session: e25eec74-eb61-43db-8633-263f865e863b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466464295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466473694Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"session_id":"e25eec74-eb61-43db-8633-263f865e863b"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466484562Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Activated sub_volume 2df9a117-0925-4254-bd50-3c3cba4b9813","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.466493151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.46650181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.46681523Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"task":"crutest"} 01/30 Write block 51 len 4096 data: 5 02/30 Read block 158 len 8192 03/30 Read block 138 len 40960 04/30 Write block 594 len 12288 data: 5 5 4 05/30 Write block 104 len 12288 data: 2 2 2 06/30 Write block 121 len 24576 data: 6 7 7 6 5 4 07/30 Write block 566 len 32768 data: 4 3 3 2 2 2 2 4 08/30 Flush 09/30 Write block 291 len 4096 data: 4 10/30 Write block 9 len 4096 data: 4 11/30 Read block 392 len 36864 12/30 Read block 355 len 20480 13/30 Write block 99 len 4096 data: 5 14/30 Read block 194 len 16384 15/30 Read block 182 len 24576 16/30 Write block 256 len 8192 data: 3 2 17/30 Write block 523 len 24576 data: 5 3 3 4 3 4 18/30 Read block 576 len 36864 19/30 Read block 19 len 28672 20/30 Write block 94 len 20480 data: 6 5 5 5 5 21/30 Flush 22/30 Write block 308 len 8192 data: 6 7 23/30 Write block 54 len 8192 data: 3 2 24/30 Read block 355 len 12288 25/30 Read block 199 len 32768 26/30 Read block 307 len 32768 27/30 Read block 15 len 36864 28/30 Write block 498 len 20480 data: 4 3 3 2 2 29/30 Write block 188 len 24576 data: 3 3 3 3 4 3 30/30 Read block 86 len 20480 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 25 26 Done 30 29 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:50:47.47493243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:50:47.475128254Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1307,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 36 74 74 72 F F F <--- 1 020-039 28 28 28 62 62 62 F F F 2 040-059 38 38 36 73 73 72 T T T <--- 3 060-079 31 31 31 65 65 65 F F F 4 080-099 38 38 28 74 74 62 F F F <--- 5 100-119 38 38 22 73 73 48 F F T <--- 6 120-139 38 38 36 73 73 72 F F F <--- 7 140-159 33 33 33 68 68 68 F F F 8 160-179 36 36 36 72 72 72 F F F 9 180-199 36 36 36 72 72 72 T T F <--- 10 200-219 34 34 34 70 70 70 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 38 38 36 74 74 72 F F F <--- 13 260-279 28 28 28 61 61 61 F F F 14 280-299 38 38 31 74 74 65 F F F <--- 15 300-319 32 32 32 67 67 67 T T F <--- 16 320-339 36 36 36 72 72 72 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 35 35 35 71 71 71 F F F 23 460-479 36 36 36 72 72 72 F F F 24 480-499 31 31 31 65 65 65 T T F <--- 25 500-519 36 36 36 72 72 72 T T F <--- 26 520-539 38 38 28 74 74 62 F F F <--- 27 540-559 36 36 36 72 72 72 F F F 28 560-579 38 38 36 73 73 72 F F F <--- 29 580-599 38 38 36 73 73 72 F F T <--- Max gen: 38, Max flush: 74 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 39 39 39 75 75 75 F F F 3 060-079 31 31 31 65 65 65 F F F 4 080-099 38 38 38 74 74 74 F F F 5 100-119 38 38 38 73 73 73 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 36 36 36 72 72 72 F F F 9 180-199 39 39 39 75 75 75 F F F 10 200-219 34 34 34 70 70 70 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 38 38 38 74 74 74 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 38 38 38 74 74 74 F F F 15 300-319 39 39 39 75 75 75 F F F 16 320-339 36 36 36 72 72 72 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 35 35 35 71 71 71 F F F 23 460-479 36 36 36 72 72 72 F F F 24 480-499 39 39 39 75 75 75 F F F 25 500-519 39 39 39 75 75 75 F F F 26 520-539 38 38 38 74 74 74 F F F 27 540-559 36 36 36 72 72 72 F F F 28 560-579 38 38 38 73 73 73 F F F 29 580-599 38 38 38 73 73 73 F F F Max gen: 39, Max flush: 75 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.756391653Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:01.764327287Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.76437758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.771888119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772178991Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772215456Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772225365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772238513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Crucible 1d3966e1-8caf-4a1f-bf4a-eac253912103 has session id: 7834240d-e0f2-4876-8f08-5f6c00ca85f0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772719821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Upstairs opts: Upstairs UUID: 1d3966e1-8caf-4a1f-bf4a-eac253912103, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772743248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Crucible stats registered with UUID: 1d3966e1-8caf-4a1f-bf4a-eac253912103","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772754647Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:01.772765375Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772921445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772940753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.772952661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773120869Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773140127Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"1d3966e1-8caf-4a1f-bf4a-eac253912103 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773175962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773186961Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"0","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773198479Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"1","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773209858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"2","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773221146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773232655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773244243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42945)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773316064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56077)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.77334644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59249)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.773359688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.77411984Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"2","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774152426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"1","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774247743Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"0","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774605487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[0]R flush_numbers[0..12]: [74, 62, 75, 65, 74, 73, 73, 68, 72, 75, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774644162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[0]R generation[0..12]: [38, 28, 39, 31, 38, 38, 38, 33, 36, 39, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774657011Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774669719Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[1]R flush_numbers[0..12]: [74, 62, 75, 65, 74, 73, 73, 68, 72, 75, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774681807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[1]R generation[0..12]: [38, 28, 39, 31, 38, 38, 38, 33, 36, 39, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774693666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774706114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[2]R flush_numbers[0..12]: [74, 62, 75, 65, 74, 73, 73, 68, 72, 75, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774718043Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[2]R generation[0..12]: [38, 28, 39, 31, 38, 38, 38, 33, 36, 39, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774729841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774742509Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774753378Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774764667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774777025Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774787794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"":"downstairs","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774798982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774809131Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"0","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774820069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"1","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774831098Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"client":"2","":"downstairs","session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774842277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774852635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"1d3966e1-8caf-4a1f-bf4a-eac253912103 is now active with session: 7834240d-e0f2-4876-8f08-5f6c00ca85f0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774863994Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774873932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"session_id":"7834240d-e0f2-4876-8f08-5f6c00ca85f0"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774884261Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Activated sub_volume 1d3966e1-8caf-4a1f-bf4a-eac253912103","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.77489341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.774902849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.775274901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} 01/30 Write block 476 len 24576 data: 3 2 2 2 2 2 02/30 Write block 249 len 24576 data: 3 4 4 4 4 2 03/30 Read block 438 len 32768 04/30 Read block 118 len 4096 05/30 Write block 171 len 32768 data: 4 3 3 4 4 5 4 4 06/30 Write block 55 len 4096 data: 3 07/30 Read block 349 len 12288 08/30 Flush 09/30 Write block 439 len 32768 data: 5 6 6 8 6 6 6 7 10/30 Read block 75 len 28672 11/30 Read block 254 len 8192 12/30 Read block 456 len 4096 13/30 Read block 532 len 28672 14/30 Read block 558 len 12288 15/30 Write block 90 len 16384 data: 3 3 4 4 16/30 Read block 312 len 28672 17/30 Write block 217 len 28672 data: 5 7 8 9 8 9 9 18/30 Read block 116 len 40960 19/30 Flush 20/30 Write block 59 len 32768 data: 2 4 5 4 4 4 3 3 21/30 Write block 309 len 40960 data: 8 6 8 5 6 6 6 4 4 3 22/30 Write block 540 len 24576 data: 7 8 9 5 3 4 23/30 Write block 215 len 32768 data: 3 4 6 8 9 10 9 10 24/30 Read block 362 len 24576 25/30 Read block 588 len 20480 26/30 Read block 87 len 40960 27/30 Write block 422 len 24576 data: 4 3 3 3 2 2 28/30 Write block 235 len 20480 data: 2 2 2 3 3 29/30 Write block 176 len 32768 data: 6 5 5 2 3 3 4 5 30/30 Write block 482 len 20480 data: 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 6 Sent Sent Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 4 3 32 Done 5 26 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:01.785052896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:01.78525604Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1316,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 39 40 40 75 76 76 F T T <--- 3 060-079 31 31 31 65 65 65 F T T <--- 4 080-099 38 40 40 74 77 77 F F F <--- 5 100-119 38 38 38 73 73 73 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 36 40 40 72 76 76 T T T <--- 9 180-199 39 39 39 75 75 75 F T T <--- 10 200-219 34 40 40 70 77 77 F T T <--- 11 220-239 34 40 40 70 77 77 F T T <--- 12 240-259 38 40 40 74 76 76 T F F <--- 13 260-279 28 28 28 61 61 61 F F F 14 280-299 38 38 38 74 74 74 F F F 15 300-319 39 39 39 75 75 75 F T T <--- 16 320-339 36 36 36 72 72 72 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 34 40 40 69 77 77 F T T <--- 22 440-459 35 40 40 71 77 77 F F F <--- 23 460-479 36 40 40 72 76 76 T F F <--- 24 480-499 39 40 40 75 76 76 T T T <--- 25 500-519 39 39 39 75 75 75 F F F 26 520-539 38 38 38 74 74 74 F F F 27 540-559 36 36 36 72 72 72 F T T <--- 28 560-579 38 38 38 73 73 73 F F F 29 580-599 38 38 38 73 73 73 F F F Max gen: 40, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 28 28 28 62 62 62 F F F 2 040-059 41 41 41 78 78 78 F F F 3 060-079 41 41 41 78 78 78 F F F 4 080-099 40 40 40 77 77 77 F F F 5 100-119 38 38 38 73 73 73 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 41 41 41 78 78 78 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 41 41 41 78 78 78 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 38 38 38 74 74 74 F F F 15 300-319 41 41 41 78 78 78 F F F 16 320-339 36 36 36 72 72 72 F F F 17 340-359 30 30 30 64 64 64 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 41 41 41 78 78 78 F F F 22 440-459 40 40 40 77 77 77 F F F 23 460-479 40 40 40 76 76 76 F F F 24 480-499 41 41 41 78 78 78 F F F 25 500-519 39 39 39 75 75 75 F F F 26 520-539 38 38 38 74 74 74 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 38 38 38 73 73 73 F F F 29 580-599 38 38 38 73 73 73 F F F Max gen: 41, Max flush: 78 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.070572024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:16.078350867Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.07840667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.085772797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.08606257Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086095666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086105074Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086117803Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Crucible 0418476f-31c5-46fe-9f95-731de7295934 has session id: a7ed8da7-8eb7-4458-b216-f50951763a8a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.08645079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Upstairs opts: Upstairs UUID: 0418476f-31c5-46fe-9f95-731de7295934, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086498064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Crucible stats registered with UUID: 0418476f-31c5-46fe-9f95-731de7295934","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086509702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:16.086519851Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086660133Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.08667823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086689309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086848668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086862996Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"0418476f-31c5-46fe-9f95-731de7295934 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086884264Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086893872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"0","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086904561Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"1","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.08691519Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"2","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086925778Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086936807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086947675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44150)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.086997969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62533)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.087027915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51567)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.087039294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.087771669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"1","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.087794846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"2","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.087807194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"0","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088173797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[0]R flush_numbers[0..12]: [74, 62, 78, 78, 77, 73, 73, 68, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088195034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[0]R generation[0..12]: [38, 28, 41, 41, 40, 38, 38, 33, 41, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088209042Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088221301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[1]R flush_numbers[0..12]: [74, 62, 78, 78, 77, 73, 73, 68, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088232939Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[1]R generation[0..12]: [38, 28, 41, 41, 40, 38, 38, 33, 41, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088244348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088256656Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[2]R flush_numbers[0..12]: [74, 62, 78, 78, 77, 73, 73, 68, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088268235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[2]R generation[0..12]: [38, 28, 41, 41, 40, 38, 38, 33, 41, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088279593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088291862Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.08830225Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088313199Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088323797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088334356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"":"downstairs","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088345095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088354923Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"0","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088365542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"1","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088377601Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"client":"2","":"downstairs","session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088388469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088398668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"0418476f-31c5-46fe-9f95-731de7295934 is now active with session: a7ed8da7-8eb7-4458-b216-f50951763a8a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088409696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088424804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"session_id":"a7ed8da7-8eb7-4458-b216-f50951763a8a"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088435093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Activated sub_volume 0418476f-31c5-46fe-9f95-731de7295934","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088443992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088453091Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.088810585Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} 01/30 Write block 332 len 24576 data: 7 7 7 6 5 3 02/30 Write block 36 len 12288 data: 2 2 2 03/30 Write block 43 len 40960 data: 4 5 7 6 6 6 7 4 6 6 04/30 Write block 384 len 24576 data: 4 4 5 6 7 9 05/30 Write block 95 len 28672 data: 6 6 6 6 6 3 2 06/30 Write block 66 len 32768 data: 4 3 3 2 2 3 3 2 07/30 Write block 366 len 36864 data: 7 7 6 6 6 4 4 3 4 08/30 Write block 204 len 4096 data: 5 09/30 Read block 160 len 32768 10/30 Read block 450 len 36864 11/30 Write block 103 len 20480 data: 2 3 3 3 2 12/30 Read block 520 len 36864 13/30 Read block 549 len 36864 14/30 Write block 426 len 24576 data: 3 3 2 2 4 5 15/30 Write block 191 len 20480 data: 4 5 4 3 3 16/30 Read block 158 len 28672 17/30 Read block 392 len 24576 18/30 Write block 519 len 24576 data: 6 6 6 6 6 4 19/30 Read block 117 len 40960 20/30 Read block 478 len 16384 21/30 Read block 274 len 16384 22/30 Read block 18 len 8192 23/30 Read block 480 len 36864 24/30 Read block 159 len 16384 25/30 Read block 214 len 32768 26/30 Read block 432 len 12288 27/30 Read block 493 len 20480 28/30 Write block 454 len 4096 data: 3 29/30 Write block 338 len 16384 data: 2 2 3 3 30/30 Read block 97 len 16384 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 3 29 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:16.097560413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:16.097795462Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1325,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 28 28 28 62 62 62 T T T 2 040-059 41 41 41 78 78 78 T T T 3 060-079 41 41 41 78 78 78 F T T <--- 4 080-099 40 40 40 77 77 77 F T T <--- 5 100-119 38 38 38 73 73 73 F T T <--- 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 41 41 41 78 78 78 F T T <--- 10 200-219 41 41 41 78 78 78 F T T <--- 11 220-239 41 41 41 78 78 78 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 38 38 38 74 74 74 F F F 15 300-319 41 41 41 78 78 78 F F F 16 320-339 36 36 36 72 72 72 T T T 17 340-359 30 30 30 64 64 64 F T T <--- 18 360-379 33 33 33 68 68 68 F T T <--- 19 380-399 36 36 36 72 72 72 F T T <--- 20 400-419 29 29 29 63 63 63 F F F 21 420-439 41 41 41 78 78 78 F T T <--- 22 440-459 40 40 40 77 77 77 F T T <--- 23 460-479 40 40 40 76 76 76 F F F 24 480-499 41 41 41 78 78 78 F F F 25 500-519 39 39 39 75 75 75 F T T <--- 26 520-539 38 38 38 74 74 74 F T T <--- 27 540-559 41 41 41 78 78 78 F F F 28 560-579 38 38 38 73 73 73 F F F 29 580-599 38 38 38 73 73 73 F F F Max gen: 41, Max flush: 78 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 341,342c341,342 < 1, < 1, --- > 2, > 2, diff found after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 42 42 42 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 42 42 42 79 79 79 F F F 10 200-219 42 42 42 79 79 79 F F F 11 220-239 41 41 41 78 78 78 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 28 28 28 61 61 61 F F F 14 280-299 38 38 38 74 74 74 F F F 15 300-319 41 41 41 78 78 78 F F F 16 320-339 42 42 42 79 79 79 F F F 17 340-359 42 42 42 79 79 79 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 42 42 42 79 79 79 F F F 20 400-419 29 29 29 63 63 63 F F F 21 420-439 42 42 42 79 79 79 F F F 22 440-459 42 42 42 79 79 79 F F F 23 460-479 40 40 40 76 76 76 F F F 24 480-499 41 41 41 78 78 78 F F F 25 500-519 42 42 42 79 79 79 F F F 26 520-539 42 42 42 79 79 79 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 38 38 38 73 73 73 F F F 29 580-599 38 38 38 73 73 73 F F F Max gen: 42, Max flush: 79 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.393409685Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403004544Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403367727Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403402073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403446177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403476763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Crucible 1f40b35c-c57d-4000-9879-dd75b6390184 has session id: 52f34a2d-3417-4628-a430-4b3b76cea24a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403841786Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Upstairs opts: Upstairs UUID: 1f40b35c-c57d-4000-9879-dd75b6390184, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403864823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Crucible stats registered with UUID: 1f40b35c-c57d-4000-9879-dd75b6390184","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.403877041Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:30.40388797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.40404379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404080255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404094023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404268291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404287248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"1f40b35c-c57d-4000-9879-dd75b6390184 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404326373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404337712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"0","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.40434958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"1","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404361299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"2","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404373037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404385055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404399424Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"ds_connection connected from Ok(127.0.0.1:32914)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.404476214Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50477)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.40450608Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55555)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.40458408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405170384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"1","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405193931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"0","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405207769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"2","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405605048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[0]R flush_numbers[0..12]: [74, 79, 79, 79, 79, 79, 73, 68, 78, 79, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405642833Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[0]R generation[0..12]: [38, 42, 42, 42, 42, 42, 38, 33, 41, 42, 42, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405656271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405669919Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[1]R flush_numbers[0..12]: [74, 79, 79, 79, 79, 79, 73, 68, 78, 79, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405682788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[1]R generation[0..12]: [38, 42, 42, 42, 42, 42, 38, 33, 41, 42, 42, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405695476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405708764Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[2]R flush_numbers[0..12]: [74, 79, 79, 79, 79, 79, 73, 68, 78, 79, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405723752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[2]R generation[0..12]: [38, 42, 42, 42, 42, 42, 38, 33, 41, 42, 42, 41]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405736481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405749749Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405761247Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405772906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405784164Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405795553Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"":"downstairs","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405807571Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.40581827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"0","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405830298Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"1","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405841987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"client":"2","":"downstairs","session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405853705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405864994Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"1f40b35c-c57d-4000-9879-dd75b6390184 is now active with session: 52f34a2d-3417-4628-a430-4b3b76cea24a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405877162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405887731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"session_id":"52f34a2d-3417-4628-a430-4b3b76cea24a"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.40589873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Activated sub_volume 1f40b35c-c57d-4000-9879-dd75b6390184","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405909678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.405919457Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.406292469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"task":"crutest"} 01/30 Write block 338 len 36864 data: 2 2 4 4 3 4 4 4 4 02/30 Write block 572 len 24576 data: 3 5 4 4 4 5 03/30 Read block 344 len 24576 04/30 Write block 185 len 12288 data: 4 5 3 05/30 Flush 06/30 Write block 406 len 12288 data: 3 4 4 07/30 Write block 572 len 40960 data: 4 6 5 5 5 6 5 6 6 3 08/30 Read block 582 len 4096 09/30 Write block 181 len 4096 data: 4 10/30 Flush 11/30 Write block 388 len 24576 data: 8 10 9 9 7 6 12/30 Flush 13/30 Flush 14/30 Write block 533 len 20480 data: 4 4 4 5 6 15/30 Read block 582 len 12288 16/30 Read block 294 len 24576 17/30 Read block 160 len 8192 18/30 Write block 481 len 20480 data: 3 3 3 3 4 19/30 Write block 479 len 24576 data: 3 3 4 4 4 4 20/30 Write block 273 len 4096 data: 5 21/30 Read block 254 len 36864 22/30 Read block 481 len 40960 23/30 Read block 370 len 24576 24/30 Write block 446 len 12288 data: 8 6 5 25/30 Write block 562 len 40960 data: 9 5 4 4 5 4 4 3 3 3 26/30 Flush 27/30 Write block 431 len 16384 data: 6 5 5 5 28/30 Read block 388 len 20480 29/30 Write block 227 len 20480 data: 9 7 4 4 4 30/30 Read block 67 len 8192 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 2 29 Done 30 3 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:30.41687094Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:30.417041398Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1337,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 42 42 42 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 44 42 44 81 79 81 F F F <--- 10 200-219 42 42 42 79 79 79 F F F 11 220-239 41 41 41 78 78 78 T F T <--- 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 28 44 84 61 84 F F F <--- 14 280-299 38 38 38 74 74 74 F F F 15 300-319 41 41 41 78 78 78 F F F 16 320-339 44 42 44 80 79 80 F T F <--- 17 340-359 44 42 44 80 79 80 F T F <--- 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 42 44 82 79 82 F F F <--- 20 400-419 44 29 44 81 63 81 F F F <--- 21 420-439 42 42 42 79 79 79 T F T <--- 22 440-459 44 42 44 84 79 84 F F F <--- 23 460-479 44 40 44 84 76 84 F F F <--- 24 480-499 44 41 44 84 78 84 F F F <--- 25 500-519 42 42 42 79 79 79 F F F 26 520-539 44 42 44 84 79 84 F F F <--- 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 38 44 84 73 84 F T F <--- 29 580-599 44 38 44 81 73 81 F F F <--- Max gen: 44, Max flush: 84 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 42 42 42 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 44 44 44 81 81 81 F F F 10 200-219 42 42 42 79 79 79 F F F 11 220-239 45 45 45 85 85 85 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 38 38 38 74 74 74 F F F 15 300-319 41 41 41 78 78 78 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 44 44 44 80 80 80 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 44 44 82 82 82 F F F 20 400-419 44 44 44 81 81 81 F F F 21 420-439 45 45 45 85 85 85 F F F 22 440-459 44 44 44 84 84 84 F F F 23 460-479 44 44 44 84 84 84 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 42 42 42 79 79 79 F F F 26 520-539 44 44 44 84 84 84 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 45, Max flush: 85 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.70230188Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:44.710550633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.710609956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.718726486Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719042485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.71907795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719088049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719101567Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Crucible 236ff44e-326c-4ba7-92ed-24b4a62a81d0 has session id: a85d9a4e-c922-48e2-8da4-f35cada33061","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719520943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Upstairs opts: Upstairs UUID: 236ff44e-326c-4ba7-92ed-24b4a62a81d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719567507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Crucible stats registered with UUID: 236ff44e-326c-4ba7-92ed-24b4a62a81d0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719584215Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:44.719595283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719745434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719764481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.71977641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719954407Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.719969465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"236ff44e-326c-4ba7-92ed-24b4a62a81d0 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.72000942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720020628Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"0","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720032027Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"1","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720043475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"2","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720054784Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720091139Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720119036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"ds_connection connected from Ok(127.0.0.1:51674)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720132334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"ds_connection connected from Ok(127.0.0.1:56346)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720145082Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"ds_connection connected from Ok(127.0.0.1:49549)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720257898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.720999222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"1","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721025318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"0","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721163311Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"2","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721458142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[0]R flush_numbers[0..12]: [74, 79, 79, 79, 79, 79, 73, 68, 78, 81, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721480839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[0]R generation[0..12]: [38, 42, 42, 42, 42, 42, 38, 33, 41, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721493368Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721506296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[1]R flush_numbers[0..12]: [74, 79, 79, 79, 79, 79, 73, 68, 78, 81, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721518735Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[1]R generation[0..12]: [38, 42, 42, 42, 42, 42, 38, 33, 41, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721530973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721543671Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[2]R flush_numbers[0..12]: [74, 79, 79, 79, 79, 79, 73, 68, 78, 81, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.72155587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[2]R generation[0..12]: [38, 42, 42, 42, 42, 42, 38, 33, 41, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721568328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721581077Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721592085Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721603344Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721616652Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.72162781Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"":"downstairs","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721639329Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721649668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"0","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721660946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"1","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721672065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"client":"2","":"downstairs","session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721683853Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721694672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 236ff44e-326c-4ba7-92ed-24b4a62a81d0 is now active with session: a85d9a4e-c922-48e2-8da4-f35cada33061","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.72171207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721722508Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"session_id":"a85d9a4e-c922-48e2-8da4-f35cada33061"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721733147Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Activated sub_volume 236ff44e-326c-4ba7-92ed-24b4a62a81d0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721742546Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.721752294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.722000972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} 01/30 Write block 472 len 12288 data: 9 8 5 02/30 Read block 134 len 16384 03/30 Write block 529 len 28672 data: 4 4 4 4 5 5 5 04/30 Read block 210 len 40960 05/30 Read block 462 len 40960 06/30 Write block 32 len 20480 data: 3 3 3 2 3 07/30 Read block 539 len 20480 08/30 Flush 09/30 Write block 446 len 32768 data: 9 7 6 3 3 2 2 2 10/30 Read block 495 len 24576 11/30 Read block 262 len 16384 12/30 Read block 294 len 4096 13/30 Read block 331 len 36864 14/30 Read block 403 len 4096 15/30 Write block 21 len 4096 data: 3 16/30 Write block 285 len 20480 data: 2 4 4 4 4 17/30 Write block 449 len 24576 data: 4 4 3 3 3 4 18/30 Write block 306 len 36864 data: 7 8 7 9 7 9 6 7 7 19/30 Read block 322 len 28672 20/30 Read block 321 len 36864 21/30 Write block 349 len 8192 data: 2 4 22/30 Read block 505 len 4096 23/30 Read block 82 len 16384 24/30 Write block 311 len 20480 data: 10 7 8 8 7 25/30 Read block 427 len 20480 26/30 Read block 553 len 12288 27/30 Read block 565 len 24576 28/30 Write block 302 len 24576 data: 3 3 5 5 8 9 29/30 Read block 143 len 36864 30/30 Read block 92 len 12288 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 4 Sent Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:44.732820493Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:44.733034055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1346,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 42 46 46 79 86 86 F T T <--- 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 42 42 42 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 44 44 44 81 81 81 F F F 10 200-219 42 42 42 79 79 79 F F F 11 220-239 45 45 45 85 85 85 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 38 38 38 74 74 74 F T T <--- 15 300-319 41 41 41 78 78 78 F T T <--- 16 320-339 44 44 44 80 80 80 F F F 17 340-359 44 44 44 80 80 80 F T T <--- 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 44 44 82 82 82 F F F 20 400-419 44 44 44 81 81 81 F F F 21 420-439 45 45 45 85 85 85 F F F 22 440-459 44 44 44 84 84 84 F T T <--- 23 460-479 44 46 46 84 86 86 T F F <--- 24 480-499 44 44 44 84 84 84 F F F 25 500-519 42 42 42 79 79 79 F F F 26 520-539 44 46 46 84 86 86 F F F <--- 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 46, Max flush: 86 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 47 47 47 87 87 87 F F F 2 040-059 42 42 42 79 79 79 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 42 42 42 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F F F 9 180-199 44 44 44 81 81 81 F F F 10 200-219 42 42 42 79 79 79 F F F 11 220-239 45 45 45 85 85 85 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 47 47 47 87 87 87 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 44 44 82 82 82 F F F 20 400-419 44 44 44 81 81 81 F F F 21 420-439 45 45 45 85 85 85 F F F 22 440-459 47 47 47 87 87 87 F F F 23 460-479 46 46 46 86 86 86 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 42 42 42 79 79 79 F F F 26 520-539 46 46 46 86 86 86 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 47, Max flush: 87 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.00775365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:59.015694613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.015752745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.023260494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.023542707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.023574763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.023583952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.023596451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Crucible 8bb8ba63-0519-4abc-8a02-c14c1d695b18 has session id: 74f7e6db-dce6-457c-8d64-4dd397227504","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024049092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Upstairs opts: Upstairs UUID: 8bb8ba63-0519-4abc-8a02-c14c1d695b18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024069459Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Crucible stats registered with UUID: 8bb8ba63-0519-4abc-8a02-c14c1d695b18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024080638Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:59.024090627Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024240017Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024258565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024269583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024442281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024458019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"8bb8ba63-0519-4abc-8a02-c14c1d695b18 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024494744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024506883Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"0","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024517581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"1","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.02452838Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"2","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024539519Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024550797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024561756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"ds_connection connected from Ok(127.0.0.1:57559)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024619118Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"ds_connection connected from Ok(127.0.0.1:55274)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024635686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"ds_connection connected from Ok(127.0.0.1:43716)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.024647295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025368291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"1","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025419135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"2","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025434373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"0","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025774499Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[0]R flush_numbers[0..12]: [74, 87, 79, 79, 79, 79, 73, 68, 78, 81, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025812064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[0]R generation[0..12]: [38, 47, 42, 42, 42, 42, 38, 33, 41, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025824282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.02583878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[1]R flush_numbers[0..12]: [74, 87, 79, 79, 79, 79, 73, 68, 78, 81, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025850839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[1]R generation[0..12]: [38, 47, 42, 42, 42, 42, 38, 33, 41, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025862577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025874846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[2]R flush_numbers[0..12]: [74, 87, 79, 79, 79, 79, 73, 68, 78, 81, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025886354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[2]R generation[0..12]: [38, 47, 42, 42, 42, 42, 38, 33, 41, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025897943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025910201Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.02592066Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025931688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025942207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025952756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"":"downstairs","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025963454Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025973083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"0","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.025983852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"1","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.02599467Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"client":"2","":"downstairs","session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026007309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026017587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"8bb8ba63-0519-4abc-8a02-c14c1d695b18 is now active with session: 74f7e6db-dce6-457c-8d64-4dd397227504","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026028716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026038465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"session_id":"74f7e6db-dce6-457c-8d64-4dd397227504"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026048623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Activated sub_volume 8bb8ba63-0519-4abc-8a02-c14c1d695b18","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026057622Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026066681Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.026388689Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} 01/30 Write block 392 len 28672 data: 8 7 7 7 6 7 5 02/30 Read block 576 len 12288 03/30 Write block 434 len 40960 data: 6 4 4 6 6 6 7 7 9 7 04/30 Read block 357 len 28672 05/30 Write block 526 len 16384 data: 5 4 5 5 06/30 Write block 185 len 36864 data: 5 6 4 4 4 4 5 6 5 07/30 Flush 08/30 Write block 50 len 12288 data: 5 7 7 09/30 Write block 295 len 8192 data: 4 2 10/30 Read block 14 len 16384 11/30 Read block 485 len 24576 12/30 Read block 162 len 40960 13/30 Write block 439 len 20480 data: 7 8 8 10 8 14/30 Flush 15/30 Write block 112 len 16384 data: 2 2 2 2 16/30 Read block 534 len 28672 17/30 Write block 468 len 20480 data: 8 9 9 9 10 18/30 Read block 235 len 8192 19/30 Read block 326 len 40960 20/30 Read block 389 len 32768 21/30 Write block 171 len 12288 data: 5 4 4 22/30 Read block 380 len 8192 23/30 Read block 369 len 12288 24/30 Write block 102 len 12288 data: 2 3 4 25/30 Read block 363 len 12288 26/30 Read block 263 len 16384 27/30 Read block 450 len 8192 28/30 Write block 35 len 32768 data: 3 4 3 3 2 3 4 4 29/30 Read block 486 len 32768 30/30 Write block 403 len 20480 data: 5 3 3 4 5 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 5 29 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:51:59.036653822Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:51:59.036855876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1355,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 47 47 47 87 87 87 F T T <--- 2 040-059 42 48 48 79 89 89 F T T <--- 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 42 42 42 79 79 79 F T T <--- 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 41 41 41 78 78 78 F T T <--- 9 180-199 44 48 48 81 88 88 F F F <--- 10 200-219 42 42 42 79 79 79 F F F 11 220-239 45 45 45 85 85 85 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 47 48 48 87 89 89 F F F <--- 15 300-319 47 47 47 87 87 87 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 44 48 48 82 88 88 T F F <--- 20 400-419 44 44 44 81 81 81 F T T <--- 21 420-439 45 48 48 85 89 89 T F F <--- 22 440-459 47 48 48 87 89 89 T F F <--- 23 460-479 46 46 46 86 86 86 F T T <--- 24 480-499 44 44 44 84 84 84 F F F 25 500-519 42 42 42 79 79 79 F F F 26 520-539 46 48 48 86 88 88 T F F <--- 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 48, Max flush: 89 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 49 49 49 90 90 90 F F F 2 040-059 49 49 49 90 90 90 F F F 3 060-079 42 42 42 79 79 79 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 49 49 49 90 90 90 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 49 49 49 90 90 90 F F F 9 180-199 48 48 48 88 88 88 F F F 10 200-219 42 42 42 79 79 79 F F F 11 220-239 45 45 45 85 85 85 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 42 42 42 79 79 79 F F F 19 380-399 48 48 48 88 88 88 F F F 20 400-419 49 49 49 90 90 90 F F F 21 420-439 48 48 48 89 89 89 F F F 22 440-459 48 48 48 89 89 89 F F F 23 460-479 49 49 49 90 90 90 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 42 42 42 79 79 79 F F F 26 520-539 48 48 48 88 88 88 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 49, Max flush: 90 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.319939679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.328908409Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329221798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329257664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329267522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329280491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Crucible 070bde8e-1b55-4262-949a-31ee8195bee5 has session id: 709095ac-0d11-48e1-9e46-438b2f3a145e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.32967324Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Upstairs opts: Upstairs UUID: 070bde8e-1b55-4262-949a-31ee8195bee5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329697017Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Crucible stats registered with UUID: 070bde8e-1b55-4262-949a-31ee8195bee5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329713215Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:13.329723433Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329885572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329918778Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.329930876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330090326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330104854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"070bde8e-1b55-4262-949a-31ee8195bee5 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330130891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330141299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"0","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330152068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"1","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330162666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"2","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330173185Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330184284Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330195202Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45977)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330245526Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40677)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330259594Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61491)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330271102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.330979241Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"1","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331025625Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"0","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331038533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"2","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.33137091Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[0]R flush_numbers[0..12]: [74, 90, 90, 79, 79, 90, 73, 68, 90, 88, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331401826Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[0]R generation[0..12]: [38, 49, 49, 42, 42, 49, 38, 33, 49, 48, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331414305Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331426593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[1]R flush_numbers[0..12]: [74, 90, 90, 79, 79, 90, 73, 68, 90, 88, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331438092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[1]R generation[0..12]: [38, 49, 49, 42, 42, 49, 38, 33, 49, 48, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.33144952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331461668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[2]R flush_numbers[0..12]: [74, 90, 90, 79, 79, 90, 73, 68, 90, 88, 79, 85]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331473237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[2]R generation[0..12]: [38, 49, 49, 42, 42, 49, 38, 33, 49, 48, 42, 45]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331484616Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331499324Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331509782Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331520461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.33153075Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331541058Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"":"downstairs","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331551657Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331561216Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"0","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331571674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"1","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331582043Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"client":"2","":"downstairs","session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331592902Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.33160303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"070bde8e-1b55-4262-949a-31ee8195bee5 is now active with session: 709095ac-0d11-48e1-9e46-438b2f3a145e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331613899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331623438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"session_id":"709095ac-0d11-48e1-9e46-438b2f3a145e"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331633456Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Activated sub_volume 070bde8e-1b55-4262-949a-31ee8195bee5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331642115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.331651094Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.332019186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"task":"crutest"} 01/30 Write block 525 len 36864 data: 4 6 5 6 6 5 5 5 6 02/30 Read block 371 len 20480 03/30 Write block 429 len 28672 data: 3 5 7 6 6 7 5 04/30 Read block 405 len 12288 05/30 Write block 237 len 8192 data: 3 4 06/30 Write block 209 len 4096 data: 6 07/30 Write block 21 len 36864 data: 4 3 3 3 3 4 4 3 4 08/30 Write block 411 len 36864 data: 2 2 2 2 2 4 4 4 4 09/30 Read block 155 len 8192 10/30 Read block 582 len 20480 11/30 Write block 113 len 12288 data: 3 3 3 12/30 Read block 363 len 16384 13/30 Flush 14/30 Write block 444 len 8192 data: 7 7 15/30 Read block 404 len 36864 16/30 Write block 514 len 32768 data: 6 5 5 9 7 7 7 7 17/30 Read block 521 len 4096 18/30 Read block 269 len 16384 19/30 Write block 54 len 40960 data: 4 4 2 2 2 3 5 6 5 5 20/30 Read block 37 len 20480 21/30 Write block 362 len 12288 data: 7 8 7 22/30 Read block 169 len 24576 23/30 Read block 545 len 16384 24/30 Write block 105 len 24576 data: 4 4 3 2 2 2 25/30 Read block 171 len 36864 26/30 Read block 432 len 12288 27/30 Write block 426 len 16384 data: 4 4 3 4 28/30 Read block 381 len 20480 29/30 Read block 305 len 4096 30/30 Read block 400 len 20480 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) None Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:13.341681947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:13.34188464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1364,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 50 49 50 91 90 91 F F F <--- 2 040-059 49 49 49 90 90 90 T F T <--- 3 060-079 42 42 42 79 79 79 T F T <--- 4 080-099 42 42 42 79 79 79 F F F 5 100-119 50 49 50 91 90 91 T F T <--- 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 49 49 49 90 90 90 F F F 9 180-199 48 48 48 88 88 88 F F F 10 200-219 50 42 50 91 79 91 F F F <--- 11 220-239 50 45 50 91 85 91 F F F <--- 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 42 42 42 79 79 79 T F T <--- 19 380-399 48 48 48 88 88 88 F F F 20 400-419 50 49 50 91 90 91 F F F <--- 21 420-439 50 48 50 91 89 91 T F T <--- 22 440-459 48 48 48 89 89 89 T F T <--- 23 460-479 49 49 49 90 90 90 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 42 42 42 79 79 79 T F T <--- 26 520-539 50 48 50 91 88 91 T F T <--- 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 50, Max flush: 91 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 51 51 51 92 92 92 F F F 3 060-079 51 51 51 92 92 92 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 51 51 51 92 92 92 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 49 49 49 90 90 90 F F F 9 180-199 48 48 48 88 88 88 F F F 10 200-219 50 50 50 91 91 91 F F F 11 220-239 50 50 50 91 91 91 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 44 44 44 80 80 80 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 51 51 51 92 92 92 F F F 19 380-399 48 48 48 88 88 88 F F F 20 400-419 50 50 50 91 91 91 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 51 51 51 92 92 92 F F F 23 460-479 49 49 49 90 90 90 F F F 24 480-499 44 44 44 84 84 84 F F F 25 500-519 51 51 51 92 92 92 F F F 26 520-539 51 51 51 92 92 92 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 44 44 44 84 84 84 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 51, Max flush: 92 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.620174014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.629198976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.629512286Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.62955403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.629564479Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.629577337Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Crucible f08425dd-1b18-4c9e-93db-0387fb6e31eb has session id: 0260f091-1af4-4ac3-8fbf-feb157e20d28","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630007842Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Upstairs opts: Upstairs UUID: f08425dd-1b18-4c9e-93db-0387fb6e31eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630039958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Crucible stats registered with UUID: f08425dd-1b18-4c9e-93db-0387fb6e31eb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630052626Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:27.630063055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630184679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630204136Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630215475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630401281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630420728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"f08425dd-1b18-4c9e-93db-0387fb6e31eb active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630449835Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630460193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"0","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630470992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"1","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630481651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"2","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630492139Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630503178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630514176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"ds_connection connected from Ok(127.0.0.1:45024)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630570619Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"ds_connection connected from Ok(127.0.0.1:38122)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630588667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"ds_connection connected from Ok(127.0.0.1:58795)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.630600465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631353218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"0","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631381294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"2","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631510747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"1","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631795711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[0]R flush_numbers[0..12]: [74, 91, 92, 92, 79, 92, 73, 68, 90, 88, 91, 91]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631833326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[0]R generation[0..12]: [38, 50, 51, 51, 42, 51, 38, 33, 49, 48, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631847404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631859692Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[1]R flush_numbers[0..12]: [74, 91, 92, 92, 79, 92, 73, 68, 90, 88, 91, 91]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631871231Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[1]R generation[0..12]: [38, 50, 51, 51, 42, 51, 38, 33, 49, 48, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631882579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631894448Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[2]R flush_numbers[0..12]: [74, 91, 92, 92, 79, 92, 73, 68, 90, 88, 91, 91]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631905816Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[2]R generation[0..12]: [38, 50, 51, 51, 42, 51, 38, 33, 49, 48, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631917095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631929073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631939502Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631950241Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631960599Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631970918Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"":"downstairs","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631981706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.631991365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"0","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632003194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"1","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632013872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"client":"2","":"downstairs","session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632024621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.6320348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"f08425dd-1b18-4c9e-93db-0387fb6e31eb is now active with session: 0260f091-1af4-4ac3-8fbf-feb157e20d28","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632046098Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632055757Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"session_id":"0260f091-1af4-4ac3-8fbf-feb157e20d28"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632065736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Activated sub_volume f08425dd-1b18-4c9e-93db-0387fb6e31eb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632074624Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.632083603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.63242156Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"task":"crutest"} 01/30 Write block 215 len 4096 data: 4 02/30 Read block 121 len 8192 03/30 Read block 368 len 40960 04/30 Write block 588 len 20480 data: 3 5 6 5 6 05/30 Write block 192 len 8192 data: 7 6 06/30 Read block 258 len 12288 07/30 Read block 540 len 12288 08/30 Write block 165 len 20480 data: 3 3 3 3 3 09/30 Write block 475 len 36864 data: 5 4 3 3 4 4 5 5 5 10/30 Read block 30 len 24576 11/30 Write block 177 len 32768 data: 6 6 3 4 5 5 6 5 12/30 Write block 410 len 20480 data: 3 3 3 3 3 13/30 Read block 3 len 8192 14/30 Read block 60 len 16384 15/30 Read block 105 len 8192 16/30 Write block 271 len 16384 data: 5 4 6 5 17/30 Read block 363 len 20480 18/30 Write block 547 len 16384 data: 4 4 4 5 19/30 Read block 392 len 36864 20/30 Read block 575 len 36864 21/30 Read block 70 len 40960 22/30 Write block 328 len 8192 data: 8 7 23/30 Read block 306 len 4096 24/30 Read block 21 len 16384 25/30 Read block 582 len 32768 26/30 Read block 547 len 32768 27/30 Write block 558 len 32768 data: 6 7 8 9 10 6 5 5 28/30 Read block 361 len 20480 29/30 Write block 410 len 40960 data: 4 4 4 4 4 3 5 5 5 5 30/30 Read block 5 len 36864 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:27.642122335Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:27.642329128Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1373,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 51 51 51 92 92 92 F F F 3 060-079 51 51 51 92 92 92 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 51 51 51 92 92 92 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 49 49 49 90 90 90 T F T <--- 9 180-199 48 48 48 88 88 88 T F T <--- 10 200-219 50 50 50 91 91 91 T T T 11 220-239 50 50 50 91 91 91 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 44 44 44 84 84 84 T F T <--- 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 44 44 44 80 80 80 T F T <--- 17 340-359 47 47 47 87 87 87 F F F 18 360-379 51 51 51 92 92 92 F F F 19 380-399 48 48 48 88 88 88 F F F 20 400-419 50 50 50 91 91 91 T F T <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 51 51 51 92 92 92 F F F 23 460-479 49 49 49 90 90 90 T F T <--- 24 480-499 44 44 44 84 84 84 T F T <--- 25 500-519 51 51 51 92 92 92 F F F 26 520-539 51 51 51 92 92 92 F F F 27 540-559 41 41 41 78 78 78 T F T <--- 28 560-579 44 44 44 84 84 84 T F T <--- 29 580-599 44 44 44 81 81 81 T F T <--- Max gen: 51, Max flush: 92 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 50 50 50 91 91 91 F F F 2 040-059 51 51 51 92 92 92 F F F 3 060-079 51 51 51 92 92 92 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 51 51 51 92 92 92 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 33 33 33 68 68 68 F F F 8 160-179 52 52 52 93 93 93 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 F F F 11 220-239 50 50 50 91 91 91 F F F 12 240-259 40 40 40 76 76 76 F F F 13 260-279 52 52 52 93 93 93 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 51 51 51 92 92 92 F F F 19 380-399 48 48 48 88 88 88 F F F 20 400-419 52 52 52 93 93 93 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 51 51 51 92 92 92 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 51 51 51 92 92 92 F F F 26 520-539 51 51 51 92 92 92 F F F 27 540-559 52 52 52 93 93 93 F F F 28 560-579 52 52 52 93 93 93 F F F 29 580-599 52 52 52 93 93 93 F F F Max gen: 52, Max flush: 93 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.934647736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:41.942684267Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.94273603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.950381961Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.950672194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.950705459Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.950714838Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.950730166Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Crucible 06f1e120-3ba8-4250-bb79-c3aae1fa6ec1 has session id: 173d10c6-74cd-437f-a0c1-18cc7d50fa11","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951069992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Upstairs opts: Upstairs UUID: 06f1e120-3ba8-4250-bb79-c3aae1fa6ec1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951151982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Crucible stats registered with UUID: 06f1e120-3ba8-4250-bb79-c3aae1fa6ec1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.95116777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:41.951177798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951325399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951359795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951372073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951527693Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951538012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"06f1e120-3ba8-4250-bb79-c3aae1fa6ec1 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951564048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951574427Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"0","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951585126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"1","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951595884Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"2","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951606473Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951619461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.95163052Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"ds_connection connected from Ok(127.0.0.1:38475)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951687092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"ds_connection connected from Ok(127.0.0.1:61749)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951698861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"ds_connection connected from Ok(127.0.0.1:52237)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.951710729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952503247Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"2","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952519035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"1","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952535033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"0","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952887577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[0]R flush_numbers[0..12]: [74, 91, 92, 92, 79, 92, 73, 68, 93, 93, 93, 91]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952909644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[0]R generation[0..12]: [38, 50, 51, 51, 42, 51, 38, 33, 52, 52, 52, 50]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952921543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952933841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[1]R flush_numbers[0..12]: [74, 91, 92, 92, 79, 92, 73, 68, 93, 93, 93, 91]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.95294555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[1]R generation[0..12]: [38, 50, 51, 51, 42, 51, 38, 33, 52, 52, 52, 50]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952957278Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952971956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[2]R flush_numbers[0..12]: [74, 91, 92, 92, 79, 92, 73, 68, 93, 93, 93, 91]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952983665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[2]R generation[0..12]: [38, 50, 51, 51, 42, 51, 38, 33, 52, 52, 52, 50]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.952995353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953007562Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.95301832Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953029259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953039757Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953050286Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"":"downstairs","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953061365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953076973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"0","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953087781Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"1","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.9530985Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"client":"2","":"downstairs","session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953109958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953120247Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"06f1e120-3ba8-4250-bb79-c3aae1fa6ec1 is now active with session: 173d10c6-74cd-437f-a0c1-18cc7d50fa11","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953131246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953141264Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"session_id":"173d10c6-74cd-437f-a0c1-18cc7d50fa11"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953152723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Activated sub_volume 06f1e120-3ba8-4250-bb79-c3aae1fa6ec1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953161762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953170811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.953480151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} 01/30 Write block 260 len 24576 data: 2 2 2 2 2 2 02/30 Read block 484 len 40960 03/30 Read block 504 len 36864 04/30 Read block 464 len 20480 05/30 Write block 402 len 8192 data: 5 6 06/30 Read block 187 len 28672 07/30 Write block 548 len 36864 data: 5 5 6 4 4 5 6 6 5 08/30 Read block 356 len 8192 09/30 Flush 10/30 Write block 244 len 40960 data: 6 5 6 5 4 4 5 5 5 5 11/30 Write block 132 len 12288 data: 4 4 4 12/30 Write block 142 len 4096 data: 4 13/30 Read block 42 len 8192 14/30 Write block 175 len 12288 data: 5 7 7 15/30 Read block 67 len 36864 16/30 Write block 127 len 4096 data: 2 17/30 Write block 62 len 8192 data: 6 6 18/30 Write block 158 len 32768 data: 4 5 5 5 6 6 5 4 19/30 Write block 535 len 4096 data: 6 20/30 Write block 32 len 20480 data: 4 4 4 4 5 21/30 Read block 11 len 16384 22/30 Read block 405 len 32768 23/30 Write block 235 len 36864 data: 3 3 4 5 4 3 5 5 5 24/30 Read block 196 len 32768 25/30 Read block 7 len 20480 26/30 Flush 27/30 Write block 107 len 36864 data: 4 3 3 3 2 3 4 4 4 28/30 Read block 304 len 12288 29/30 Read block 297 len 12288 30/30 Read block 300 len 12288 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:41.963804765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:41.964024577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1382,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 50 54 54 91 95 95 F F F <--- 2 040-059 51 51 51 92 92 92 F F F 3 060-079 51 54 54 92 95 95 F F F <--- 4 080-099 42 42 42 79 79 79 F F F 5 100-119 51 51 51 92 92 92 F T T <--- 6 120-139 38 54 54 73 95 95 F F F <--- 7 140-159 33 54 54 68 95 95 F F F <--- 8 160-179 52 54 54 93 95 95 F F F <--- 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 F F F 11 220-239 50 54 54 91 95 95 F F F <--- 12 240-259 40 54 54 76 95 95 F F F <--- 13 260-279 52 54 54 93 94 94 T F F <--- 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 51 51 51 92 92 92 F F F 19 380-399 48 48 48 88 88 88 F F F 20 400-419 52 54 54 93 94 94 F F F <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 51 51 51 92 92 92 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 51 51 51 92 92 92 F F F 26 520-539 51 54 54 92 95 95 F F F <--- 27 540-559 52 54 54 93 94 94 F F F <--- 28 560-579 52 52 52 93 93 93 F F F 29 580-599 52 52 52 93 93 93 F F F Max gen: 54, Max flush: 95 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 51 51 51 92 92 92 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 54 54 54 95 95 95 F F F 8 160-179 54 54 54 95 95 95 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 54 54 54 94 94 94 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 52 52 52 93 93 93 F F F 17 340-359 47 47 47 87 87 87 F F F 18 360-379 51 51 51 92 92 92 F F F 19 380-399 48 48 48 88 88 88 F F F 20 400-419 54 54 54 94 94 94 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 51 51 51 92 92 92 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 51 51 51 92 92 92 F F F 26 520-539 54 54 54 95 95 95 F F F 27 540-559 54 54 54 94 94 94 F F F 28 560-579 52 52 52 93 93 93 F F F 29 580-599 52 52 52 93 93 93 F F F Max gen: 55, Max flush: 96 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.247681096Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.256733645Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257064192Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257104317Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257114756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257127884Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Crucible e9248145-41ff-4d87-ae9f-3e28730382cd has session id: 6475aa2d-2ca6-4cd3-ad05-63804973526a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257567577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Upstairs opts: Upstairs UUID: e9248145-41ff-4d87-ae9f-3e28730382cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.25762054Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Crucible stats registered with UUID: e9248145-41ff-4d87-ae9f-3e28730382cd","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257636298Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:56.257646607Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257787469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257822704Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257834993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257984183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.257998901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"e9248145-41ff-4d87-ae9f-3e28730382cd active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258025328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258035887Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"0","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258047025Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"1","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258057904Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"2","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258068552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258079801Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258091059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36715)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.25816173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44633)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258218923Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60752)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258275416Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258852871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"1","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258886127Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"0","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.258977075Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"2","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259263508Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[0]R flush_numbers[0..12]: [74, 95, 92, 95, 79, 96, 95, 95, 95, 93, 93, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259293444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[0]R generation[0..12]: [38, 54, 51, 54, 42, 55, 54, 54, 54, 52, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259305752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259318161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[1]R flush_numbers[0..12]: [74, 95, 92, 95, 79, 96, 95, 95, 95, 93, 93, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259329799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[1]R generation[0..12]: [38, 54, 51, 54, 42, 55, 54, 54, 54, 52, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259341618Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259353866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[2]R flush_numbers[0..12]: [74, 95, 92, 95, 79, 96, 95, 95, 95, 93, 93, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259365615Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[2]R generation[0..12]: [38, 54, 51, 54, 42, 55, 54, 54, 54, 52, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259377043Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259389172Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.25939968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259410679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259421167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259433166Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"":"downstairs","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259444064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259453853Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"0","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259464652Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"1","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.25947532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"client":"2","":"downstairs","session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259486449Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259496688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"e9248145-41ff-4d87-ae9f-3e28730382cd is now active with session: 6475aa2d-2ca6-4cd3-ad05-63804973526a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259507886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259523764Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"session_id":"6475aa2d-2ca6-4cd3-ad05-63804973526a"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259534033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Activated sub_volume e9248145-41ff-4d87-ae9f-3e28730382cd","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259543092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.25955214Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.259782621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"task":"crutest"} 01/30 Write block 458 len 8192 data: 5 6 02/30 Read block 261 len 24576 03/30 Read block 47 len 32768 04/30 Read block 579 len 28672 05/30 Read block 523 len 40960 06/30 Write block 586 len 4096 data: 3 07/30 Write block 326 len 12288 data: 6 7 9 08/30 Write block 280 len 36864 data: 2 2 2 2 2 3 5 5 5 09/30 Read block 550 len 4096 10/30 Read block 187 len 4096 11/30 Write block 388 len 20480 data: 9 11 10 10 9 12/30 Read block 92 len 20480 13/30 Flush 14/30 Flush 15/30 Write block 524 len 20480 data: 5 5 7 6 7 16/30 Write block 288 len 4096 data: 6 17/30 Flush 18/30 Write block 358 len 12288 data: 5 5 5 19/30 Write block 270 len 36864 data: 6 6 5 7 6 5 4 3 2 20/30 Read block 347 len 16384 21/30 Read block 444 len 12288 22/30 Write block 540 len 32768 data: 8 9 10 6 4 5 4 5 23/30 Flush 24/30 Write block 212 len 4096 data: 3 25/30 Read block 231 len 32768 26/30 Read block 468 len 8192 27/30 Write block 350 len 8192 data: 5 4 28/30 Write block 168 len 4096 data: 4 29/30 Read block 315 len 36864 30/30 Read block 191 len 40960 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:52:56.269170246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:52:56.26937765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1391,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 51 51 51 92 92 92 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 54 54 54 95 95 95 F F F 8 160-179 54 54 54 95 95 95 T T F <--- 9 180-199 52 52 52 93 93 93 F F F 10 200-219 52 52 52 93 93 93 T T F <--- 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 54 100 100 94 F F F <--- 14 280-299 56 56 48 99 99 89 F F F <--- 15 300-319 47 47 47 87 87 87 F F F 16 320-339 56 56 52 97 97 93 F F F <--- 17 340-359 56 56 47 100 100 87 T T F <--- 18 360-379 56 56 51 100 100 92 F F F <--- 19 380-399 56 56 48 97 97 88 F F F <--- 20 400-419 54 54 54 94 94 94 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 56 56 51 97 97 92 F F T <--- 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 51 51 51 92 92 92 F F F 26 520-539 56 56 54 99 99 95 F F F <--- 27 540-559 56 56 54 100 100 94 F F F <--- 28 560-579 52 52 52 93 93 93 F F F 29 580-599 56 56 52 97 97 93 F F F <--- Max gen: 56, Max flush: 100 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 51 51 51 92 92 92 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 42 42 42 79 79 79 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 54 54 54 95 95 95 F F F 7 140-159 54 54 54 95 95 95 F F F 8 160-179 57 57 57 101 101 101 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 57 57 57 101 101 101 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 56 100 100 100 F F F 14 280-299 56 56 56 99 99 99 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 56 56 56 97 97 97 F F F 17 340-359 57 57 57 101 101 101 F F F 18 360-379 56 56 56 100 100 100 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 54 54 54 94 94 94 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 F F F 25 500-519 51 51 51 92 92 92 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 56 56 56 100 100 100 F F F 28 560-579 52 52 52 93 93 93 F F F 29 580-599 56 56 56 97 97 97 F F F Max gen: 57, Max flush: 101 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.558111042Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.567566809Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.567889817Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.567924963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.567937281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.5679506Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Crucible d2ae9858-9da4-495f-ab5b-2a4de625c21a has session id: 61b2d967-40ff-46dd-a118-1f098e9b6dfb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568343859Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Upstairs opts: Upstairs UUID: d2ae9858-9da4-495f-ab5b-2a4de625c21a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568377584Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Crucible stats registered with UUID: d2ae9858-9da4-495f-ab5b-2a4de625c21a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568390403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:10.568401251Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.56856291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568597946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568632301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.56880133Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568817268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"d2ae9858-9da4-495f-ab5b-2a4de625c21a active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568858702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568869751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"0","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568880979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"1","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568892148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"2","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568905496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568917105Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568928703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56766)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568984736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:35554)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.568997384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:54770)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.569009403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.569669747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"1","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.569717001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"0","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.569810029Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"2","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570084454Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[0]R flush_numbers[0..12]: [74, 95, 92, 95, 79, 96, 95, 95, 101, 93, 101, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570107411Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[0]R generation[0..12]: [38, 54, 51, 54, 42, 55, 54, 54, 57, 52, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570119319Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570132037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[1]R flush_numbers[0..12]: [74, 95, 92, 95, 79, 96, 95, 95, 101, 93, 101, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570144016Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[1]R generation[0..12]: [38, 54, 51, 54, 42, 55, 54, 54, 57, 52, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570157924Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570170552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[2]R flush_numbers[0..12]: [74, 95, 92, 95, 79, 96, 95, 95, 101, 93, 101, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570182341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[2]R generation[0..12]: [38, 54, 51, 54, 42, 55, 54, 54, 57, 52, 57, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570194229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570206788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570217696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570228825Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570239594Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570250512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"":"downstairs","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570261831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570271989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"0","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570283138Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"1","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570294236Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"client":"2","":"downstairs","session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570305545Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570316114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"d2ae9858-9da4-495f-ab5b-2a4de625c21a is now active with session: 61b2d967-40ff-46dd-a118-1f098e9b6dfb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570327662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570338921Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"session_id":"61b2d967-40ff-46dd-a118-1f098e9b6dfb"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570349469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Activated sub_volume d2ae9858-9da4-495f-ab5b-2a4de625c21a","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-05-14T20:53:10.570358958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.570371906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.57064999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"task":"crutest"} 01/30 Write block 218 len 4096 data: 9 02/30 Read block 318 len 24576 03/30 Write block 132 len 36864 data: 5 5 5 4 4 3 3 5 5 04/30 Read block 200 len 24576 05/30 Write block 40 len 36864 data: 4 5 5 5 6 8 7 7 7 06/30 Write block 337 len 20480 data: 4 3 3 5 5 07/30 Write block 546 len 36864 data: 5 6 6 6 7 5 5 6 7 08/30 Write block 162 len 32768 data: 7 7 6 5 4 4 5 4 09/30 Write block 591 len 24576 data: 6 7 6 6 6 5 10/30 Write block 508 len 28672 data: 5 5 5 9 8 7 7 11/30 Read block 445 len 36864 12/30 Read block 448 len 16384 13/30 Flush 14/30 Write block 213 len 28672 data: 4 3 5 5 7 10 10 15/30 Write block 84 len 28672 data: 3 4 4 4 4 2 4 16/30 Read block 379 len 20480 17/30 Flush 18/30 Write block 156 len 28672 data: 3 4 5 6 6 6 8 19/30 Read block 441 len 8192 20/30 Write block 488 len 16384 data: 2 3 5 5 21/30 Write block 273 len 32768 data: 8 7 6 5 4 3 2 3 22/30 Read block 177 len 8192 23/30 Write block 540 len 40960 data: 9 10 11 7 5 6 6 7 7 7 24/30 Read block 195 len 16384 25/30 Write block 556 len 20480 data: 6 5 7 8 9 26/30 Read block 488 len 32768 27/30 Write block 401 len 32768 data: 4 6 7 4 4 5 6 5 28/30 Read block 89 len 40960 29/30 Write block 86 len 16384 data: 5 5 5 3 30/30 Read block 65 len 36864 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:10.581343237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:10.581560439Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1400,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 58 58 51 102 102 92 F F F <--- 3 060-079 54 54 54 95 95 95 F F F 4 080-099 58 58 42 103 103 79 T T F <--- 5 100-119 55 55 55 96 96 96 F F F 6 120-139 58 58 54 102 102 95 F F F <--- 7 140-159 58 58 54 102 102 95 T T F <--- 8 160-179 58 58 57 102 102 101 T T F <--- 9 180-199 52 52 52 93 93 93 F F F 10 200-219 58 58 57 103 103 101 F F F <--- 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 56 56 56 100 100 100 T T F <--- 14 280-299 56 56 56 99 99 99 T T F <--- 15 300-319 47 47 47 87 87 87 F F F 16 320-339 58 58 56 102 102 97 F F F <--- 17 340-359 58 58 57 102 102 101 F F F <--- 18 360-379 56 56 56 100 100 100 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 54 54 54 94 94 94 T T F <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 52 52 52 93 93 93 T T F <--- 25 500-519 58 58 51 102 102 92 F F F <--- 26 520-539 56 56 56 99 99 99 F F F 27 540-559 58 58 56 102 102 100 T T F <--- 28 560-579 52 52 52 93 93 93 T T F <--- 29 580-599 58 58 56 102 102 97 F F F <--- Max gen: 58, Max flush: 103 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 59 59 59 104 104 104 F F F 5 100-119 55 55 55 96 96 96 F F F 6 120-139 58 58 58 102 102 102 F F F 7 140-159 59 59 59 104 104 104 F F F 8 160-179 59 59 59 104 104 104 F F F 9 180-199 52 52 52 93 93 93 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 54 54 95 95 95 F F F 13 260-279 59 59 59 104 104 104 F F F 14 280-299 59 59 59 104 104 104 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 58 58 58 102 102 102 F F F 17 340-359 58 58 58 102 102 102 F F F 18 360-379 56 56 56 100 100 100 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 59 59 59 104 104 104 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 52 52 52 93 93 93 F F F 24 480-499 59 59 59 104 104 104 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 59 59 59 104 104 104 F F F 28 560-579 59 59 59 104 104 104 F F F 29 580-599 58 58 58 102 102 102 F F F Max gen: 59, Max flush: 104 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.880681188Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:24.888832913Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.888889426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.896697976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897010105Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89705101Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897062159Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897076047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Crucible f47b6d34-9d7f-445b-bed0-88c118057e10 has session id: bff2ed4a-11e7-47e5-8d28-67f81156fbcc","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897456738Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Upstairs opts: Upstairs UUID: f47b6d34-9d7f-445b-bed0-88c118057e10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897500282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Crucible stats registered with UUID: f47b6d34-9d7f-445b-bed0-88c118057e10","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89751891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:24.897529858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89766737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897686198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897698117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897894571Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897913699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"f47b6d34-9d7f-445b-bed0-88c118057e10 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897945934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897956763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"0","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897967852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"1","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89797907Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"2","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.897989969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898001597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898013076Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57873)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898073288Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:49746)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898087346Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43728)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898103414Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898866405Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"1","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898973162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"2","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.898999508Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"0","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899535189Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[0]R flush_numbers[0..12]: [74, 95, 102, 95, 104, 96, 102, 104, 104, 93, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899575994Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[0]R generation[0..12]: [38, 54, 58, 54, 59, 55, 58, 59, 59, 52, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899588992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89960198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[1]R flush_numbers[0..12]: [74, 95, 102, 95, 104, 96, 102, 104, 104, 93, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899614789Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[1]R generation[0..12]: [38, 54, 58, 54, 59, 55, 58, 59, 59, 52, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899626997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899639825Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[2]R flush_numbers[0..12]: [74, 95, 102, 95, 104, 96, 102, 104, 104, 93, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899651974Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[2]R generation[0..12]: [38, 54, 58, 54, 59, 55, 58, 59, 59, 52, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899664222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89967988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899691019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899702477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899713416Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899724464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"":"downstairs","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899735793Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899746102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"0","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89975736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"1","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899768719Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"client":"2","":"downstairs","session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899780137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899790946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"f47b6d34-9d7f-445b-bed0-88c118057e10 is now active with session: bff2ed4a-11e7-47e5-8d28-67f81156fbcc","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899802524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899812943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"session_id":"bff2ed4a-11e7-47e5-8d28-67f81156fbcc"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.899823462Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Activated sub_volume f47b6d34-9d7f-445b-bed0-88c118057e10","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.89983278Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-05-14T20:53:24.899848648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.900137191Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} 01/30 Write block 256 len 32768 data: 4 3 2 2 3 3 3 3 02/30 Write block 279 len 12288 data: 3 4 3 03/30 Write block 191 len 12288 data: 6 8 7 04/30 Write block 275 len 8192 data: 7 6 05/30 Read block 59 len 12288 06/30 Write block 253 len 4096 data: 6 07/30 Read block 425 len 28672 08/30 Read block 252 len 36864 09/30 Write block 599 len 4096 data: 3 10/30 Read block 370 len 32768 11/30 Write block 180 len 32768 data: 5 6 6 7 6 6 7 5 12/30 Write block 360 len 20480 data: 6 4 8 9 8 13/30 Write block 449 len 8192 data: 5 5 14/30 Write block 288 len 40960 data: 7 5 4 5 4 4 4 5 3 2 15/30 Read block 482 len 28672 16/30 Read block 305 len 16384 17/30 Write block 582 len 40960 data: 3 4 4 5 4 3 4 6 7 7 18/30 Write block 368 len 24576 data: 7 7 7 5 5 4 19/30 Write block 103 len 24576 data: 4 5 5 5 5 4 20/30 Read block 421 len 12288 21/30 Read block 485 len 28672 22/30 Flush 23/30 Flush 24/30 Write block 458 len 40960 data: 6 7 6 5 4 4 6 6 6 7 25/30 Write block 461 len 20480 data: 6 5 5 7 7 26/30 Write block 409 len 12288 data: 3 5 5 27/30 Read block 64 len 4096 28/30 Read block 188 len 36864 29/30 Write block 96 len 28672 data: 7 7 7 7 4 3 3 30/30 Read block 28 len 20480 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:24.910101492Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:24.910330943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1409,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 59 59 59 104 104 104 F T T <--- 5 100-119 55 60 60 96 105 105 F T T <--- 6 120-139 58 58 58 102 102 102 F F F 7 140-159 59 59 59 104 104 104 F F F 8 160-179 59 59 59 104 104 104 F F F 9 180-199 52 60 60 93 105 105 F F F <--- 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 54 60 60 95 105 105 T F F <--- 13 260-279 59 60 60 104 105 105 T F F <--- 14 280-299 59 60 60 104 105 105 F F F <--- 15 300-319 47 47 47 87 87 87 F F F 16 320-339 58 58 58 102 102 102 F F F 17 340-359 58 58 58 102 102 102 F F F 18 360-379 56 60 60 100 105 105 F F F <--- 19 380-399 56 56 56 97 97 97 F F F 20 400-419 59 59 59 104 104 104 F T T <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 56 60 60 97 105 105 F T T <--- 23 460-479 52 52 52 93 93 93 F T T <--- 24 480-499 59 59 59 104 104 104 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 59 59 59 104 104 104 F F F 28 560-579 59 59 59 104 104 104 F F F 29 580-599 58 60 60 102 105 105 F F F <--- Max gen: 60, Max flush: 105 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 61 61 61 106 106 106 F F F 5 100-119 61 61 61 106 106 106 F F F 6 120-139 58 58 58 102 102 102 F F F 7 140-159 59 59 59 104 104 104 F F F 8 160-179 59 59 59 104 104 104 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 60 60 60 105 105 105 F F F 13 260-279 60 60 60 105 105 105 F F F 14 280-299 60 60 60 105 105 105 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 58 58 58 102 102 102 F F F 17 340-359 58 58 58 102 102 102 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 61 61 61 106 106 106 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 59 59 59 104 104 104 F F F 25 500-519 58 58 58 102 102 102 F F F 26 520-539 56 56 56 99 99 99 F F F 27 540-559 59 59 59 104 104 104 F F F 28 560-579 59 59 59 104 104 104 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 61, Max flush: 106 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.196559119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20567144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20598807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206040383Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206051591Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20606494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Crucible 8df604e9-008f-4a29-af2c-151e856750db has session id: f89d8c26-463c-4da9-86e4-a162a3b4fa30","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206487165Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Upstairs opts: Upstairs UUID: 8df604e9-008f-4a29-af2c-151e856750db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206519341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Crucible stats registered with UUID: 8df604e9-008f-4a29-af2c-151e856750db","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206532349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:39.206542908Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206661932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206685429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206697178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206888033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206907071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"8df604e9-008f-4a29-af2c-151e856750db active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206942496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206953035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"0","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206963963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"1","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206974792Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"2","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20698572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.206997119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.207021046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"ds_connection connected from Ok(127.0.0.1:36347)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.207053602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"ds_connection connected from Ok(127.0.0.1:65145)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20706593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"ds_connection connected from Ok(127.0.0.1:62667)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.207077539Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.207795796Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"1","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.207862707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"0","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.207989201Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"2","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208314269Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[0]R flush_numbers[0..12]: [74, 95, 102, 95, 106, 106, 102, 104, 104, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208342955Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[0]R generation[0..12]: [38, 54, 58, 54, 61, 61, 58, 59, 59, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208356153Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208368802Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[1]R flush_numbers[0..12]: [74, 95, 102, 95, 106, 106, 102, 104, 104, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20838087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[1]R generation[0..12]: [38, 54, 58, 54, 61, 61, 58, 59, 59, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208392768Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208405137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[2]R flush_numbers[0..12]: [74, 95, 102, 95, 106, 106, 102, 104, 104, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208417155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[2]R generation[0..12]: [38, 54, 58, 54, 61, 61, 58, 59, 59, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208428854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208441142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208451871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208462809Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208473268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208483807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"":"downstairs","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208494995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208506514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"0","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208517742Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"1","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208528731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"client":"2","":"downstairs","session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208539969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208550328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"8df604e9-008f-4a29-af2c-151e856750db is now active with session: f89d8c26-463c-4da9-86e4-a162a3b4fa30","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208561547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208571425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"session_id":"f89d8c26-463c-4da9-86e4-a162a3b4fa30"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208581654Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Activated sub_volume 8df604e9-008f-4a29-af2c-151e856750db","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208590623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.208599672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.20892452Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"task":"crutest"} 01/30 Write block 497 len 32768 data: 3 5 4 4 3 3 3 3 02/30 Write block 84 len 8192 data: 4 5 03/30 Flush 04/30 Write block 173 len 12288 data: 5 5 6 05/30 Read block 272 len 24576 06/30 Read block 201 len 24576 07/30 Write block 166 len 24576 data: 5 5 6 5 3 6 08/30 Write block 527 len 8192 data: 7 8 09/30 Read block 67 len 20480 10/30 Read block 147 len 24576 11/30 Write block 559 len 32768 data: 9 10 10 11 7 6 6 6 12/30 Read block 431 len 20480 13/30 Write block 309 len 36864 data: 10 8 11 8 9 9 8 5 5 14/30 Read block 515 len 24576 15/30 Read block 420 len 28672 16/30 Flush 17/30 Write block 93 len 4096 data: 5 18/30 Write block 242 len 12288 data: 6 6 7 19/30 Read block 384 len 8192 20/30 Read block 49 len 28672 21/30 Read block 318 len 36864 22/30 Write block 299 len 20480 data: 2 2 2 4 4 23/30 Write block 575 len 8192 data: 6 6 24/30 Read block 77 len 16384 25/30 Read block 389 len 28672 26/30 Write block 23 len 16384 data: 4 4 4 5 27/30 Write block 533 len 12288 data: 7 6 7 28/30 Read block 428 len 24576 29/30 Write block 283 len 36864 data: 3 3 4 6 6 8 6 5 6 30/30 Read block 543 len 28672 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 12 13 Done 14 13 2 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:39.218871193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:39.219074167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1418,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 54 54 54 95 95 95 T T F <--- 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 62 62 62 107 107 107 T T T 5 100-119 61 61 61 106 106 106 F F F 6 120-139 58 58 58 102 102 102 F F F 7 140-159 59 59 59 104 104 104 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 60 60 60 105 105 105 T T T 13 260-279 60 60 60 105 105 105 F F F 14 280-299 60 60 60 105 105 105 T T F <--- 15 300-319 62 62 62 108 108 108 T T F <--- 16 320-339 58 58 58 102 102 102 F F F 17 340-359 58 58 58 102 102 102 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 61 61 61 106 106 106 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 62 62 62 107 107 107 F F F 26 520-539 62 62 62 108 108 108 T T F <--- 27 540-559 62 62 62 108 108 108 F F F 28 560-579 62 62 62 108 108 108 T T F <--- 29 580-599 60 60 60 105 105 105 F F F Max gen: 62, Max flush: 108 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 63 63 63 109 109 109 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 61 61 61 106 106 106 F F F 6 120-139 58 58 58 102 102 102 F F F 7 140-159 59 59 59 104 104 104 F F F 8 160-179 62 62 62 108 108 108 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 63 63 63 109 109 109 F F F 13 260-279 60 60 60 105 105 105 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 63 63 63 109 109 109 F F F 16 320-339 58 58 58 102 102 102 F F F 17 340-359 58 58 58 102 102 102 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 61 61 61 106 106 106 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 62 62 62 107 107 107 F F F 26 520-539 63 63 63 109 109 109 F F F 27 540-559 62 62 62 108 108 108 F F F 28 560-579 63 63 63 109 109 109 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 63, Max flush: 109 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.488249739Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.497552716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.497880493Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.497914499Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.497924348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.497937866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Crucible 8c6dd36c-389e-4fcc-8598-a760b8124cf5 has session id: d62348f0-2736-4df1-b76c-23970c882b12","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498312368Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Upstairs opts: Upstairs UUID: 8c6dd36c-389e-4fcc-8598-a760b8124cf5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498329905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Crucible stats registered with UUID: 8c6dd36c-389e-4fcc-8598-a760b8124cf5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498343674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:53.498354332Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498493494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498512972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.49852464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498703137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498717925Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"8c6dd36c-389e-4fcc-8598-a760b8124cf5 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498764679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498775518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"0","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498786896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"1","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498798235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"2","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498809343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498820962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498875365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:51233)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498888123Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:59065)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.498904761Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:34103)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.499042883Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.499640046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"0","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.499675631Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"2","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.49984059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"1","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500170627Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[0]R flush_numbers[0..12]: [74, 109, 102, 95, 109, 106, 102, 104, 108, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500194764Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[0]R generation[0..12]: [38, 63, 58, 54, 63, 61, 58, 59, 62, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500207423Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500220461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[1]R flush_numbers[0..12]: [74, 109, 102, 95, 109, 106, 102, 104, 108, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500233009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[1]R generation[0..12]: [38, 63, 58, 54, 63, 61, 58, 59, 62, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500245118Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500257876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[2]R flush_numbers[0..12]: [74, 109, 102, 95, 109, 106, 102, 104, 108, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500270274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[2]R generation[0..12]: [38, 63, 58, 54, 63, 61, 58, 59, 62, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500282453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500295271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500308969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500320428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500331366Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500342305Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"":"downstairs","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500354014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500364282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"0","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500376021Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"1","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500387399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"client":"2","":"downstairs","session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500399108Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500409936Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"8c6dd36c-389e-4fcc-8598-a760b8124cf5 is now active with session: d62348f0-2736-4df1-b76c-23970c882b12","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500421565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500431703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"session_id":"d62348f0-2736-4df1-b76c-23970c882b12"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500442412Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Activated sub_volume 8c6dd36c-389e-4fcc-8598-a760b8124cf5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.500451651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.50046111Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.50084203Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"task":"crutest"} 01/30 Write block 154 len 24576 data: 2 2 4 5 6 7 02/30 Write block 522 len 4096 data: 7 03/30 Read block 267 len 20480 04/30 Read block 65 len 20480 05/30 Write block 23 len 28672 data: 5 5 5 6 5 4 5 06/30 Write block 133 len 28672 data: 6 6 5 5 4 4 6 07/30 Write block 308 len 20480 data: 8 11 9 12 9 08/30 Read block 429 len 8192 09/30 Flush 10/30 Flush 11/30 Flush 12/30 Write block 560 len 32768 data: 11 11 12 8 7 7 7 5 13/30 Flush 14/30 Write block 154 len 36864 data: 3 3 5 6 7 8 7 7 9 15/30 Read block 350 len 4096 16/30 Flush 17/30 Write block 405 len 24576 data: 5 6 7 6 4 6 18/30 Read block 394 len 20480 19/30 Write block 100 len 8192 data: 5 4 20/30 Read block 553 len 28672 21/30 Flush 22/30 Write block 364 len 24576 data: 9 6 8 8 8 8 23/30 Read block 279 len 28672 24/30 Flush 25/30 Write block 254 len 40960 data: 3 2 5 4 3 3 4 4 4 4 26/30 Write block 329 len 4096 data: 8 27/30 Read block 334 len 20480 28/30 Read block 501 len 4096 29/30 Read block 413 len 4096 30/30 Write block 519 len 36864 data: 8 8 8 8 7 6 6 8 8 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:53:53.511394566Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:53:53.51159271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1427,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 64 63 64 110 109 110 F F F <--- 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 64 61 64 115 106 115 F F F <--- 6 120-139 64 58 64 110 102 110 F F F <--- 7 140-159 64 59 64 114 104 114 F F F <--- 8 160-179 64 62 64 114 108 114 F F F <--- 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 63 63 63 109 109 109 T F T <--- 13 260-279 60 60 60 105 105 105 T F T <--- 14 280-299 63 63 63 109 109 109 F F F 15 300-319 64 63 64 110 109 110 F F F <--- 16 320-339 58 58 58 102 102 102 T F T <--- 17 340-359 58 58 58 102 102 102 F F F 18 360-379 64 60 64 116 105 116 F F F <--- 19 380-399 56 56 56 97 97 97 F F F 20 400-419 64 61 64 115 106 115 F F F <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 61 61 61 106 106 106 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 62 62 62 107 107 107 T F T <--- 26 520-539 64 63 64 110 109 110 T F T <--- 27 540-559 62 62 62 108 108 108 F F F 28 560-579 64 63 64 113 109 113 F F F <--- 29 580-599 60 60 60 105 105 105 F F F Max gen: 64, Max flush: 116 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F F F 1 020-039 64 64 64 110 110 110 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 64 64 64 115 115 115 F F F 6 120-139 64 64 64 110 110 110 F F F 7 140-159 64 64 64 114 114 114 F F F 8 160-179 64 64 64 114 114 114 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 65 65 65 117 117 117 F F F 13 260-279 65 65 65 117 117 117 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 64 64 64 110 110 110 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 58 58 58 102 102 102 F F F 18 360-379 64 64 64 116 116 116 F F F 19 380-399 56 56 56 97 97 97 F F F 20 400-419 64 64 64 115 115 115 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 61 61 61 106 106 106 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 65 65 65 117 117 117 F F F 27 540-559 62 62 62 108 108 108 F F F 28 560-579 64 64 64 113 113 113 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 65, Max flush: 117 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.799046038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:07.807518602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.807579744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.815694895Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816004474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816041049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816051128Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816065006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Crucible ae9f7cc5-ec8b-4a78-b793-c97808c4df9d has session id: 3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816529586Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Upstairs opts: Upstairs UUID: ae9f7cc5-ec8b-4a78-b793-c97808c4df9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816564032Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Crucible stats registered with UUID: ae9f7cc5-ec8b-4a78-b793-c97808c4df9d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.81657733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:07.816588738Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.81673125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816750987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816763206Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.816944193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.81695978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"ae9f7cc5-ec8b-4a78-b793-c97808c4df9d active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817000595Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817012194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"0","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817024132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"1","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817035801Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"2","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817047359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817085174Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817113181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57912)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817126649Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44464)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817139367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50784)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817152556Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817873422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"1","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817911277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"0","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.817984638Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"2","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818316785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[0]R flush_numbers[0..12]: [74, 110, 102, 95, 109, 115, 110, 114, 114, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818344071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[0]R generation[0..12]: [38, 64, 58, 54, 63, 64, 64, 64, 64, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.8183579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818371428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[1]R flush_numbers[0..12]: [74, 110, 102, 95, 109, 115, 110, 114, 114, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818383946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[1]R generation[0..12]: [38, 64, 58, 54, 63, 64, 64, 64, 64, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818396255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818409183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[2]R flush_numbers[0..12]: [74, 110, 102, 95, 109, 115, 110, 114, 114, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818421651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[2]R generation[0..12]: [38, 64, 58, 54, 63, 64, 64, 64, 64, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.81843379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818446968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818458297Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818469885Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818480894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818492222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"":"downstairs","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818503821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818515359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"0","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818526878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"1","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818538336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"client":"2","":"downstairs","session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818550085Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818561023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"ae9f7cc5-ec8b-4a78-b793-c97808c4df9d is now active with session: 3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818573042Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-05-14T20:54:07.81858347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"session_id":"3dfd08e5-b791-45ee-ad5a-b42fcabe5eb1"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818598619Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Activated sub_volume ae9f7cc5-ec8b-4a78-b793-c97808c4df9d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818608187Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818617906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.818874903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} 01/30 Write block 546 len 24576 data: 7 8 8 8 8 6 02/30 Read block 164 len 24576 03/30 Write block 253 len 32768 data: 7 4 3 6 5 4 4 5 04/30 Write block 171 len 12288 data: 7 5 6 05/30 Read block 470 len 4096 06/30 Write block 445 len 12288 data: 8 10 8 07/30 Write block 346 len 20480 data: 5 4 2 3 6 08/30 Read block 410 len 12288 09/30 Write block 414 len 24576 data: 5 4 6 6 6 6 10/30 Read block 565 len 16384 11/30 Read block 365 len 24576 12/30 Write block 145 len 8192 data: 3 3 13/30 Read block 296 len 28672 14/30 Read block 419 len 8192 15/30 Read block 495 len 28672 16/30 Write block 47 len 20480 data: 8 8 8 6 8 17/30 Read block 4 len 20480 18/30 Write block 471 len 36864 data: 10 11 9 6 6 5 4 4 5 19/30 Flush 20/30 Write block 4 len 24576 data: 5 6 5 5 5 5 21/30 Read block 266 len 8192 22/30 Read block 362 len 20480 23/30 Write block 106 len 40960 data: 6 6 5 4 4 3 4 5 5 5 24/30 Read block 347 len 12288 25/30 Write block 380 len 28672 data: 5 6 7 5 5 5 6 26/30 Write block 457 len 36864 data: 4 7 8 7 7 6 6 8 8 27/30 Read block 259 len 32768 28/30 Read block 365 len 20480 29/30 Read block 389 len 36864 30/30 Read block 82 len 28672 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 4 30 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:07.830352898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:07.83057424Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1436,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 74 74 74 F T T <--- 1 020-039 64 64 64 110 110 110 F F F 2 040-059 58 66 66 102 118 118 F F F <--- 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 64 64 64 115 115 115 F T T <--- 6 120-139 64 64 64 110 110 110 F F F 7 140-159 64 66 66 114 118 118 F F F <--- 8 160-179 64 66 66 114 118 118 T F F <--- 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 65 66 66 117 118 118 T F F <--- 13 260-279 65 66 66 117 118 118 T F F <--- 14 280-299 63 63 63 109 109 109 F F F 15 300-319 64 64 64 110 110 110 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 58 66 66 102 118 118 F F F <--- 18 360-379 64 64 64 116 116 116 F F F 19 380-399 56 56 56 97 97 97 F T T <--- 20 400-419 64 66 66 115 118 118 F F F <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 61 66 66 106 118 118 F T T <--- 23 460-479 61 66 66 106 118 118 F T T <--- 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 65 65 65 117 117 117 F F F 27 540-559 62 66 66 108 118 118 T F F <--- 28 560-579 64 64 64 113 113 113 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 66, Max flush: 118 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 119 119 119 F F F 1 020-039 64 64 64 110 110 110 F F F 2 040-059 66 66 66 118 118 118 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 67 67 67 119 119 119 F F F 6 120-139 64 64 64 110 110 110 F F F 7 140-159 66 66 66 118 118 118 F F F 8 160-179 66 66 66 118 118 118 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 54 54 95 95 95 F F F 12 240-259 66 66 66 118 118 118 F F F 13 260-279 66 66 66 118 118 118 F F F 14 280-299 63 63 63 109 109 109 F F F 15 300-319 64 64 64 110 110 110 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 66 66 66 118 118 118 F F F 18 360-379 64 64 64 116 116 116 F F F 19 380-399 67 67 67 119 119 119 F F F 20 400-419 66 66 66 118 118 118 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 65 65 65 117 117 117 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 64 64 64 113 113 113 F F F 29 580-599 60 60 60 105 105 105 F F F Max gen: 67, Max flush: 119 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.120900906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:22.129109954Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.129172306Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137054647Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137359797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137393833Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137405721Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137418719Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Crucible cf618ead-5003-46ba-a758-5fbf59e3f9e9 has session id: a763b3bd-fe05-482f-8031-24486b8ed81e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.13780445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Upstairs opts: Upstairs UUID: cf618ead-5003-46ba-a758-5fbf59e3f9e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137828406Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Crucible stats registered with UUID: cf618ead-5003-46ba-a758-5fbf59e3f9e9","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.137839805Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:22.137849934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138006503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138024421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.13803564Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138203968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138215116Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"cf618ead-5003-46ba-a758-5fbf59e3f9e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138235174Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138245063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"0","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138255891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"1","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.13826678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"2","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138278958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138290147Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138328682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45463)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138349179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43733)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138435448Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39849)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.138538715Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139017553Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"1","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139140287Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"2","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139165344Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"0","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139461015Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[0]R flush_numbers[0..12]: [119, 110, 118, 95, 109, 119, 110, 118, 118, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139490002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[0]R generation[0..12]: [67, 64, 66, 54, 63, 67, 64, 66, 66, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.13950253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139515308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[1]R flush_numbers[0..12]: [119, 110, 118, 95, 109, 119, 110, 118, 118, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139527427Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[1]R generation[0..12]: [67, 64, 66, 54, 63, 67, 64, 66, 66, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139541645Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139554073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[2]R flush_numbers[0..12]: [119, 110, 118, 95, 109, 119, 110, 118, 118, 105, 103, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139566102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[2]R generation[0..12]: [67, 64, 66, 54, 63, 67, 64, 66, 66, 60, 58, 54]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.13957778Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139590139Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139600837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139611706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139622104Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139632673Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"":"downstairs","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139643602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.13965345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"0","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139664289Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"1","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139675358Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"client":"2","":"downstairs","session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139686456Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139696765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"cf618ead-5003-46ba-a758-5fbf59e3f9e9 is now active with session: a763b3bd-fe05-482f-8031-24486b8ed81e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139710573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139720432Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"session_id":"a763b3bd-fe05-482f-8031-24486b8ed81e"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.1397306Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Activated sub_volume cf618ead-5003-46ba-a758-5fbf59e3f9e9","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139740039Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.139749348Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.140061468Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} 01/30 Write block 224 len 12288 data: 10 11 9 02/30 Flush 03/30 Write block 301 len 8192 data: 3 5 04/30 Write block 6 len 36864 data: 6 6 6 6 5 5 4 3 3 05/30 Write block 592 len 28672 data: 8 7 7 7 6 4 4 06/30 Read block 149 len 4096 07/30 Flush 08/30 Write block 286 len 36864 data: 7 7 9 7 6 7 5 5 5 09/30 Read block 180 len 24576 10/30 Read block 216 len 36864 11/30 Write block 133 len 28672 data: 7 7 6 6 5 5 7 12/30 Write block 300 len 8192 data: 3 4 13/30 Read block 161 len 40960 14/30 Read block 470 len 28672 15/30 Read block 417 len 8192 16/30 Flush 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 389 len 8192 data: 12 11 21/30 Read block 478 len 40960 22/30 Read block 373 len 28672 23/30 Write block 13 len 32768 data: 4 4 4 4 4 4 3 3 24/30 Write block 520 len 16384 data: 9 9 9 8 25/30 Write block 530 len 36864 data: 6 6 6 8 7 8 6 7 9 26/30 Read block 230 len 8192 27/30 Write block 31 len 16384 data: 3 5 5 5 28/30 Write block 358 len 40960 data: 6 6 7 5 9 10 10 7 9 9 29/30 Read block 470 len 12288 30/30 Read block 549 len 8192 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:22.14901476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:22.149206805Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1445,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 68 68 119 121 121 F T T <--- 1 020-039 64 64 64 110 110 110 F T T <--- 2 040-059 66 66 66 118 118 118 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 67 67 67 119 119 119 F F F 6 120-139 64 68 68 110 122 122 F F F <--- 7 140-159 66 66 66 118 118 118 F F F 8 160-179 66 66 66 118 118 118 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 54 68 68 95 120 120 T F F <--- 12 240-259 66 66 66 118 118 118 F F F 13 260-279 66 66 66 118 118 118 F F F 14 280-299 63 68 68 109 122 122 F F F <--- 15 300-319 64 68 68 110 122 122 F F F <--- 16 320-339 65 65 65 117 117 117 F F F 17 340-359 66 66 66 118 118 118 F T T <--- 18 360-379 64 64 64 116 116 116 F T T <--- 19 380-399 67 67 67 119 119 119 F T T <--- 20 400-419 66 66 66 118 118 118 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 65 65 65 117 117 117 F T T <--- 27 540-559 66 66 66 118 118 118 F F F 28 560-579 64 64 64 113 113 113 F F F 29 580-599 60 68 68 105 121 121 F F F <--- Max gen: 68, Max flush: 122 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 123 123 123 F F F 1 020-039 69 69 69 123 123 123 F F F 2 040-059 66 66 66 118 118 118 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 67 67 67 119 119 119 F F F 6 120-139 68 68 68 122 122 122 F F F 7 140-159 66 66 66 118 118 118 F F F 8 160-179 66 66 66 118 118 118 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 58 58 58 103 103 103 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 66 66 66 118 118 118 F F F 13 260-279 66 66 66 118 118 118 F F F 14 280-299 68 68 68 122 122 122 F F F 15 300-319 68 68 68 122 122 122 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 69 69 69 123 123 123 F F F 19 380-399 69 69 69 123 123 123 F F F 20 400-419 66 66 66 118 118 118 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 69 69 69 123 123 123 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 64 64 64 113 113 113 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 69, Max flush: 123 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.431214597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:36.439399259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.439452002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.447312865Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.447611756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.447646162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.447656201Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.447669569Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Crucible 91ed5c52-7cb7-41fb-abef-1e8ef3862d19 has session id: 09cc5712-4f36-4a04-9ca5-12f0b5cddcef","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448055349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Upstairs opts: Upstairs UUID: 91ed5c52-7cb7-41fb-abef-1e8ef3862d19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.44812577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Crucible stats registered with UUID: 91ed5c52-7cb7-41fb-abef-1e8ef3862d19","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448145707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:36.448156696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448302747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448322034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448334033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448522689Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448537377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"91ed5c52-7cb7-41fb-abef-1e8ef3862d19 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448574442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.44858558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"0","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448597059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"1","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448608537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"2","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448619926Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448631904Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448643703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"ds_connection connected from Ok(127.0.0.1:49600)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448691277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"ds_connection connected from Ok(127.0.0.1:37953)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448724432Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"ds_connection connected from Ok(127.0.0.1:54427)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.448776546Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449462847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"1","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449493423Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"0","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449579892Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"2","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449907429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[0]R flush_numbers[0..12]: [123, 123, 118, 95, 109, 119, 122, 118, 118, 105, 103, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449954363Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[0]R generation[0..12]: [69, 69, 66, 54, 63, 67, 68, 66, 66, 60, 58, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449967542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.44998053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[1]R flush_numbers[0..12]: [123, 123, 118, 95, 109, 119, 122, 118, 118, 105, 103, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.449993338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[1]R generation[0..12]: [69, 69, 66, 54, 63, 67, 68, 66, 66, 60, 58, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450005587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450018585Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[2]R flush_numbers[0..12]: [123, 123, 118, 95, 109, 119, 122, 118, 118, 105, 103, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450031143Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[2]R generation[0..12]: [69, 69, 66, 54, 63, 67, 68, 66, 66, 60, 58, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450043272Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.45005863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450069878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450081357Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450092266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450103314Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"":"downstairs","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450115003Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450125571Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"0","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.45013731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"1","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450148648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"client":"2","":"downstairs","session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450160157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450170725Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"91ed5c52-7cb7-41fb-abef-1e8ef3862d19 is now active with session: 09cc5712-4f36-4a04-9ca5-12f0b5cddcef","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450182494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450192843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"session_id":"09cc5712-4f36-4a04-9ca5-12f0b5cddcef"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450203631Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Activated sub_volume 91ed5c52-7cb7-41fb-abef-1e8ef3862d19","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.45021304Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450222659Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.450590151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} 01/30 Write block 413 len 24576 data: 5 6 5 7 7 7 02/30 Write block 204 len 16384 data: 6 5 4 6 03/30 Write block 31 len 4096 data: 4 04/30 Flush 05/30 Write block 385 len 12288 data: 6 7 7 06/30 Write block 369 len 20480 data: 9 8 6 6 5 07/30 Write block 395 len 20480 data: 8 7 8 6 4 08/30 Read block 337 len 16384 09/30 Write block 564 len 40960 data: 8 8 8 6 5 4 4 4 5 7 10/30 Read block 114 len 40960 11/30 Read block 158 len 24576 12/30 Write block 593 len 4096 data: 8 13/30 Write block 379 len 16384 data: 5 6 7 8 14/30 Read block 437 len 36864 15/30 Flush 16/30 Write block 172 len 8192 data: 6 7 17/30 Read block 267 len 16384 18/30 Read block 117 len 8192 19/30 Flush 20/30 Write block 34 len 32768 data: 6 5 6 4 4 3 5 6 21/30 Read block 299 len 40960 22/30 Read block 266 len 12288 23/30 Read block 478 len 28672 24/30 Write block 377 len 40960 data: 5 4 6 7 8 9 6 6 7 8 25/30 Flush 26/30 Write block 405 len 24576 data: 6 7 8 7 5 7 27/30 Read block 137 len 32768 28/30 Read block 306 len 24576 29/30 Write block 149 len 24576 data: 3 3 3 2 2 4 30/30 Write block 52 len 8192 data: 8 5 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 3 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 0 24 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:36.460912126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:36.461150015Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1454,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 123 123 123 F F F 1 020-039 70 70 70 124 127 127 F F F <--- 2 040-059 66 70 70 118 127 127 F T T <--- 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 67 67 67 119 119 119 F F F 6 120-139 68 68 68 122 122 122 F F F 7 140-159 66 66 66 118 118 118 F T T <--- 8 160-179 66 70 70 118 126 126 F F F <--- 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 66 66 66 118 118 118 F F F 13 260-279 66 66 66 118 118 118 F F F 14 280-299 68 68 68 122 122 122 F F F 15 300-319 68 68 68 122 122 122 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 69 70 70 123 127 127 F F F <--- 19 380-399 69 70 70 123 127 127 F F F <--- 20 400-419 70 70 70 124 124 124 F T T <--- 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 69 69 69 123 123 123 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 64 70 70 113 125 125 F F F <--- 29 580-599 68 70 70 121 125 125 F F F <--- Max gen: 70, Max flush: 127 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 123 123 123 F F F 1 020-039 70 70 70 127 127 127 F F F 2 040-059 71 71 71 128 128 128 F F F 3 060-079 54 54 54 95 95 95 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 67 67 67 119 119 119 F F F 6 120-139 68 68 68 122 122 122 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 70 70 70 126 126 126 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 66 66 66 118 118 118 F F F 13 260-279 66 66 66 118 118 118 F F F 14 280-299 68 68 68 122 122 122 F F F 15 300-319 68 68 68 122 122 122 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 70 70 70 127 127 127 F F F 19 380-399 70 70 70 127 127 127 F F F 20 400-419 71 71 71 128 128 128 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 65 65 117 117 117 F F F 26 520-539 69 69 69 123 123 123 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 70 70 70 125 125 125 F F F 29 580-599 70 70 70 125 125 125 F F F Max gen: 71, Max flush: 128 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.744746982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:50.753150345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.753209098Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761101287Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761405677Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761440573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761450712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.76146431Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Crucible 2be3e052-3ab8-47f4-85d7-b80176fbb61a has session id: e01c325d-30af-448c-ab93-3f686af41216","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761810235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Upstairs opts: Upstairs UUID: 2be3e052-3ab8-47f4-85d7-b80176fbb61a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761883606Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Crucible stats registered with UUID: 2be3e052-3ab8-47f4-85d7-b80176fbb61a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.761896084Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:50.761906803Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762041065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762060253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762073651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762272425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762288183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"2be3e052-3ab8-47f4-85d7-b80176fbb61a active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762328728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762339897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"0","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762351405Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"1","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762362804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"2","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762374102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762385711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.7624693Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"ds_connection connected from Ok(127.0.0.1:51215)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762489677Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"ds_connection connected from Ok(127.0.0.1:41040)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762502406Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"ds_connection connected from Ok(127.0.0.1:65511)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.762638168Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763225472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"1","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763268637Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"2","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763359935Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"0","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763653477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[0]R flush_numbers[0..12]: [123, 127, 128, 95, 109, 119, 122, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763683603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[0]R generation[0..12]: [69, 70, 71, 54, 63, 67, 68, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763696491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.76370937Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[1]R flush_numbers[0..12]: [123, 127, 128, 95, 109, 119, 122, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763721768Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[1]R generation[0..12]: [69, 70, 71, 54, 63, 67, 68, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763733916Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763746495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[2]R flush_numbers[0..12]: [123, 127, 128, 95, 109, 119, 122, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763758863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[2]R generation[0..12]: [69, 70, 71, 54, 63, 67, 68, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763770702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.76378331Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763794309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763805647Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763816516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763827664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"":"downstairs","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763840283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763850631Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"0","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.76386194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"1","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763873228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"client":"2","":"downstairs","session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763885067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763895935Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"2be3e052-3ab8-47f4-85d7-b80176fbb61a is now active with session: e01c325d-30af-448c-ab93-3f686af41216","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763907564Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763917813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"session_id":"e01c325d-30af-448c-ab93-3f686af41216"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763928441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Activated sub_volume 2be3e052-3ab8-47f4-85d7-b80176fbb61a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.76393806Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.763947669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.764343897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} 01/30 Write block 61 len 40960 data: 7 7 7 5 4 5 4 4 3 3 02/30 Read block 305 len 36864 03/30 Write block 569 len 12288 data: 5 5 5 04/30 Read block 541 len 12288 05/30 Flush 06/30 Write block 506 len 8192 data: 4 4 07/30 Write block 308 len 16384 data: 9 12 10 13 08/30 Read block 164 len 20480 09/30 Write block 516 len 36864 data: 6 10 8 9 10 10 10 9 7 10/30 Write block 35 len 4096 data: 6 11/30 Write block 297 len 40960 data: 3 2 3 4 5 6 5 6 6 9 12/30 Read block 420 len 20480 13/30 Write block 274 len 16384 data: 8 8 7 5 14/30 Write block 57 len 24576 data: 3 3 4 6 8 8 15/30 Read block 526 len 16384 16/30 Read block 219 len 40960 17/30 Read block 305 len 40960 18/30 Write block 527 len 16384 data: 9 9 7 7 19/30 Read block 144 len 4096 20/30 Write block 377 len 12288 data: 6 5 7 21/30 Write block 115 len 28672 data: 6 3 4 4 5 5 7 22/30 Read block 511 len 28672 23/30 Flush 24/30 Write block 294 len 32768 data: 6 6 4 4 3 4 5 6 25/30 Read block 435 len 8192 26/30 Write block 15 len 20480 data: 5 5 5 5 4 27/30 Read block 446 len 40960 28/30 Write block 244 len 24576 data: 8 6 7 6 5 5 29/30 Read block 184 len 36864 30/30 Read block 204 len 16384 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Read 9 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:54:50.77499375Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:54:50.775205033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1463,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 123 123 123 F T T <--- 1 020-039 70 72 72 127 130 130 F F F <--- 2 040-059 71 72 72 128 130 130 F F F <--- 3 060-079 54 72 72 95 130 130 F F F <--- 4 080-099 63 63 63 109 109 109 F F F 5 100-119 67 72 72 119 130 130 F F F <--- 6 120-139 68 72 72 122 130 130 F F F <--- 7 140-159 71 71 71 128 128 128 F F F 8 160-179 70 70 70 126 126 126 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 66 66 66 118 118 118 F T T <--- 13 260-279 66 72 72 118 130 130 F F F <--- 14 280-299 68 72 72 122 130 130 F T T <--- 15 300-319 68 72 72 122 130 130 F T T <--- 16 320-339 65 65 65 117 117 117 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 70 72 72 127 130 130 F F F <--- 19 380-399 70 70 70 127 127 127 F F F 20 400-419 71 71 71 128 128 128 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 65 72 72 117 130 130 F F F <--- 26 520-539 69 72 72 123 130 130 F F F <--- 27 540-559 66 66 66 118 118 118 F F F 28 560-579 70 72 72 125 129 129 F F F <--- 29 580-599 70 70 70 125 125 125 F F F Max gen: 72, Max flush: 130 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 131 131 131 F F F 1 020-039 72 72 72 130 130 130 F F F 2 040-059 72 72 72 130 130 130 F F F 3 060-079 72 72 72 130 130 130 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 70 70 70 126 126 126 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 73 73 73 131 131 131 F F F 13 260-279 72 72 72 130 130 130 F F F 14 280-299 73 73 73 131 131 131 F F F 15 300-319 73 73 73 131 131 131 F F F 16 320-339 65 65 65 117 117 117 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 72 72 72 130 130 130 F F F 19 380-399 70 70 70 127 127 127 F F F 20 400-419 71 71 71 128 128 128 F F F 21 420-439 51 51 51 92 92 92 F F F 22 440-459 67 67 67 119 119 119 F F F 23 460-479 67 67 67 119 119 119 F F F 24 480-499 62 62 62 107 107 107 F F F 25 500-519 72 72 72 130 130 130 F F F 26 520-539 72 72 72 130 130 130 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 70 70 70 125 125 125 F F F Max gen: 73, Max flush: 131 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.062804291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.071463621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.071772851Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.071803417Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.071814846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.071827134Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Crucible 77aaa7b3-6839-43be-a976-518024022670 has session id: c9ad71ae-350b-4d42-be27-5ea0dbee159a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072239641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Upstairs opts: Upstairs UUID: 77aaa7b3-6839-43be-a976-518024022670, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072272057Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Crucible stats registered with UUID: 77aaa7b3-6839-43be-a976-518024022670","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072290964Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:05.072301183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072373254Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072406149Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072417978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072577667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072590805Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"77aaa7b3-6839-43be-a976-518024022670 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072617122Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072627271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"0","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072637849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"1","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072648308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"2","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072661016Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072671915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072682574Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56057)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072727808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50927)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072739566Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57951)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.072760024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073453114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"0","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073479431Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"2","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073572788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"1","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073878759Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[0]R flush_numbers[0..12]: [131, 130, 130, 130, 109, 130, 130, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073918064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[0]R generation[0..12]: [73, 72, 72, 72, 63, 72, 72, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073930112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073942101Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[1]R flush_numbers[0..12]: [131, 130, 130, 130, 109, 130, 130, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073953769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[1]R generation[0..12]: [73, 72, 72, 72, 63, 72, 72, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073966727Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073978586Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[2]R flush_numbers[0..12]: [131, 130, 130, 130, 109, 130, 130, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.073989924Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[2]R generation[0..12]: [73, 72, 72, 72, 63, 72, 72, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074000993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074012842Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.07402307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074033819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074043917Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074054236Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"":"downstairs","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074064655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074074194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"0","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074084642Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"1","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074095031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"client":"2","":"downstairs","session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.07410558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074115598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"77aaa7b3-6839-43be-a976-518024022670 is now active with session: c9ad71ae-350b-4d42-be27-5ea0dbee159a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074132886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074142985Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074151304Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"session_id":"c9ad71ae-350b-4d42-be27-5ea0dbee159a"} {"msg":"Activated sub_volume 77aaa7b3-6839-43be-a976-518024022670","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074161162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074170011Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.074447005Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"task":"crutest"} 01/30 Write block 513 len 20480 data: 8 8 6 7 11 02/30 Read block 308 len 28672 03/30 Write block 436 len 36864 data: 5 7 7 8 9 9 11 9 8 04/30 Write block 64 len 36864 data: 6 5 6 5 5 4 4 4 4 05/30 Flush 06/30 Write block 328 len 8192 data: 10 9 07/30 Write block 39 len 24576 data: 4 6 7 6 6 7 08/30 Read block 16 len 40960 09/30 Write block 479 len 40960 data: 6 5 6 6 6 5 5 4 2 3 10/30 Write block 282 len 8192 data: 3 4 11/30 Write block 292 len 24576 data: 6 6 7 7 5 5 12/30 Write block 61 len 40960 data: 9 9 8 7 6 7 6 6 5 5 13/30 Read block 283 len 8192 14/30 Read block 163 len 16384 15/30 Read block 370 len 12288 16/30 Flush 17/30 Write block 263 len 24576 data: 5 3 3 4 5 5 18/30 Read block 32 len 40960 19/30 Write block 504 len 40960 data: 4 3 5 5 6 6 6 10 9 9 20/30 Write block 24 len 8192 data: 6 6 21/30 Read block 172 len 20480 22/30 Read block 180 len 24576 23/30 Read block 444 len 28672 24/30 Write block 22 len 8192 data: 4 6 25/30 Write block 46 len 32768 data: 8 9 9 9 7 9 9 6 26/30 Read block 184 len 12288 27/30 Write block 289 len 32768 data: 8 7 8 7 7 8 8 6 28/30 Read block 529 len 28672 29/30 Read block 142 len 12288 30/30 Read block 248 len 28672 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:05.084414336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:05.084626419Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1472,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 131 131 131 F F F 1 020-039 74 72 74 133 130 133 T F T <--- 2 040-059 74 72 74 133 130 133 T F T <--- 3 060-079 74 72 74 133 130 133 F F F <--- 4 080-099 63 63 63 109 109 109 F F F 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 70 70 70 126 126 126 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 73 73 73 131 131 131 F F F 13 260-279 72 72 72 130 130 130 T F T <--- 14 280-299 74 73 74 133 131 133 T F T <--- 15 300-319 73 73 73 131 131 131 F F F 16 320-339 74 65 74 133 117 133 F F F <--- 17 340-359 69 69 69 123 123 123 F F F 18 360-379 72 72 72 130 130 130 F F F 19 380-399 70 70 70 127 127 127 F F F 20 400-419 71 71 71 128 128 128 F F F 21 420-439 74 51 74 132 92 132 F F F <--- 22 440-459 74 67 74 132 119 132 F F F <--- 23 460-479 74 67 74 133 119 133 F F F <--- 24 480-499 74 62 74 133 107 133 F F F <--- 25 500-519 74 72 74 132 130 132 T F T <--- 26 520-539 72 72 72 130 130 130 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 70 70 70 125 125 125 F F F Max gen: 74, Max flush: 133 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 131 131 131 F F F 1 020-039 75 75 75 134 134 134 F F F 2 040-059 75 75 75 134 134 134 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 70 70 70 126 126 126 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 73 73 73 131 131 131 F F F 13 260-279 75 75 75 134 134 134 F F F 14 280-299 75 75 75 134 134 134 F F F 15 300-319 73 73 73 131 131 131 F F F 16 320-339 74 74 74 133 133 133 F F F 17 340-359 69 69 69 123 123 123 F F F 18 360-379 72 72 72 130 130 130 F F F 19 380-399 70 70 70 127 127 127 F F F 20 400-419 71 71 71 128 128 128 F F F 21 420-439 74 74 74 132 132 132 F F F 22 440-459 74 74 74 132 132 132 F F F 23 460-479 74 74 74 133 133 133 F F F 24 480-499 74 74 74 133 133 133 F F F 25 500-519 75 75 75 134 134 134 F F F 26 520-539 72 72 72 130 130 130 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 70 70 70 125 125 125 F F F Max gen: 75, Max flush: 134 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.356496983Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.365658018Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.365975087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366009922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366019531Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366032629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Crucible 0daf045b-d261-4a0d-927d-eb084f983749 has session id: 8be30571-4dcd-4d95-b272-d553a8e94664","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366469783Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Upstairs opts: Upstairs UUID: 0daf045b-d261-4a0d-927d-eb084f983749, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366503268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Crucible stats registered with UUID: 0daf045b-d261-4a0d-927d-eb084f983749","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366518016Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:19.366528195Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366662978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366697293Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366709792Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366904116Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366920434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"0daf045b-d261-4a0d-927d-eb084f983749 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366963759Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366974437Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"0","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366985586Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"1","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.366996595Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"2","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367007663Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367019072Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.36703026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"ds_connection connected from Ok(127.0.0.1:34892)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367113159Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"ds_connection connected from Ok(127.0.0.1:35468)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367133707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"ds_connection connected from Ok(127.0.0.1:34517)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367147855Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367836736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"0","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.367864182Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"2","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.3678856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"1","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368240264Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[0]R flush_numbers[0..12]: [131, 134, 134, 133, 109, 130, 130, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.3682695Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[0]R generation[0..12]: [73, 75, 75, 74, 63, 72, 72, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368281928Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368294377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[1]R flush_numbers[0..12]: [131, 134, 134, 133, 109, 130, 130, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368306555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[1]R generation[0..12]: [73, 75, 75, 74, 63, 72, 72, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368318344Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368330842Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[2]R flush_numbers[0..12]: [131, 134, 134, 133, 109, 130, 130, 128, 126, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.36834299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[2]R generation[0..12]: [73, 75, 75, 74, 63, 72, 72, 71, 70, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368354979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368367277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368380476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368391514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368402123Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368412751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"":"downstairs","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.36842381Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368433799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"0","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368444657Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"1","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368455546Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"client":"2","":"downstairs","session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368466654Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368477113Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"0daf045b-d261-4a0d-927d-eb084f983749 is now active with session: 8be30571-4dcd-4d95-b272-d553a8e94664","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368488322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.36849825Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"session_id":"8be30571-4dcd-4d95-b272-d553a8e94664"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368508719Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Activated sub_volume 0daf045b-d261-4a0d-927d-eb084f983749","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368518128Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.368527377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.36880912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"task":"crutest"} 01/30 Flush 02/30 Write block 170 len 8192 data: 4 8 03/30 Read block 3 len 28672 04/30 Flush 05/30 Write block 283 len 24576 data: 5 4 5 8 8 10 06/30 Write block 317 len 8192 data: 6 4 07/30 Read block 312 len 4096 08/30 Write block 471 len 32768 data: 11 12 10 7 7 6 5 5 09/30 Read block 286 len 12288 10/30 Write block 467 len 8192 data: 8 9 11/30 Write block 11 len 4096 data: 6 12/30 Read block 508 len 4096 13/30 Read block 312 len 40960 14/30 Read block 425 len 32768 15/30 Write block 259 len 24576 data: 5 6 5 5 6 4 16/30 Read block 514 len 8192 17/30 Read block 408 len 36864 18/30 Write block 316 len 12288 data: 6 7 5 19/30 Read block 421 len 36864 20/30 Write block 32 len 8192 data: 6 6 21/30 Read block 476 len 4096 22/30 Write block 357 len 36864 data: 4 7 7 8 6 10 11 11 8 23/30 Write block 396 len 12288 data: 8 9 7 24/30 Write block 362 len 32768 data: 11 12 12 9 10 10 9 10 25/30 Read block 542 len 32768 26/30 Write block 531 len 4096 data: 7 27/30 Read block 19 len 40960 28/30 Read block 312 len 4096 29/30 Write block 417 len 8192 data: 8 8 30/30 Read block 30 len 28672 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 16 1 17 Done 26 10 25 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:19.377254318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:19.377464021Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1481,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 131 131 131 T T T 1 020-039 75 75 75 134 134 134 T F T <--- 2 040-059 75 75 75 134 134 134 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 76 76 76 136 136 136 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 73 73 73 131 131 131 T F T <--- 13 260-279 75 75 75 134 134 134 T F T <--- 14 280-299 75 75 75 134 134 134 T T T 15 300-319 73 73 73 131 131 131 T T T 16 320-339 74 74 74 133 133 133 F F F 17 340-359 69 69 69 123 123 123 T F T <--- 18 360-379 72 72 72 130 130 130 T F T <--- 19 380-399 70 70 70 127 127 127 T F T <--- 20 400-419 71 71 71 128 128 128 T F T <--- 21 420-439 74 74 74 132 132 132 F F F 22 440-459 74 74 74 132 132 132 F F F 23 460-479 74 74 74 133 133 133 T T T 24 480-499 74 74 74 133 133 133 F F F 25 500-519 75 75 75 134 134 134 F F F 26 520-539 72 72 72 130 130 130 T F T <--- 27 540-559 66 66 66 118 118 118 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 70 70 70 125 125 125 F F F Max gen: 76, Max flush: 136 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 77 77 77 137 137 137 F F F 2 040-059 75 75 75 134 134 134 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 63 63 63 109 109 109 F F F 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 76 76 76 136 136 136 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 68 68 68 120 120 120 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 77 77 77 137 137 137 F F F 14 280-299 77 77 77 137 137 137 F F F 15 300-319 77 77 77 137 137 137 F F F 16 320-339 74 74 74 133 133 133 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 77 77 77 137 137 137 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 77 77 77 137 137 137 F F F 21 420-439 74 74 74 132 132 132 F F F 22 440-459 74 74 74 132 132 132 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 74 74 74 133 133 133 F F F 25 500-519 75 75 75 134 134 134 F F F 26 520-539 77 77 77 137 137 137 F F F 27 540-559 66 66 66 118 118 118 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 70 70 70 125 125 125 F F F Max gen: 77, Max flush: 137 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.687003322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696012257Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696332735Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696366401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.69637591Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696389318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Crucible 947b4a65-8ea8-41bc-bd01-3da0025fa536 has session id: b8eaa205-d349-42ba-89d7-66b598b77197","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696817433Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Upstairs opts: Upstairs UUID: 947b4a65-8ea8-41bc-bd01-3da0025fa536, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696876005Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Crucible stats registered with UUID: 947b4a65-8ea8-41bc-bd01-3da0025fa536","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.696888083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:33.696898502Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697046353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697064411Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697075509Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697263245Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697278443Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"947b4a65-8ea8-41bc-bd01-3da0025fa536 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697316008Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697328596Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"0","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697339685Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"1","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697350704Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"2","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697361542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697372901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697394538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"ds_connection connected from Ok(127.0.0.1:39175)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697436263Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"ds_connection connected from Ok(127.0.0.1:45015)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697481047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"ds_connection connected from Ok(127.0.0.1:63986)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.697494825Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698284513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"0","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.69830544Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"1","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698317818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"2","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698714977Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 134, 133, 109, 130, 130, 128, 136, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698752442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[0]R generation[0..12]: [77, 77, 75, 74, 63, 72, 72, 71, 76, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698764861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698779669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 134, 133, 109, 130, 130, 128, 136, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698791457Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[1]R generation[0..12]: [77, 77, 75, 74, 63, 72, 72, 71, 76, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698803096Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698815114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 134, 133, 109, 130, 130, 128, 136, 105, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698826893Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[2]R generation[0..12]: [77, 77, 75, 74, 63, 72, 72, 71, 76, 60, 70, 68]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698838181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.69885028Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698860718Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698871457Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698881906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698892514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"":"downstairs","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698903603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698913451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"0","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.69892422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"1","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698936029Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"client":"2","":"downstairs","session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698947237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698957536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"947b4a65-8ea8-41bc-bd01-3da0025fa536 is now active with session: b8eaa205-d349-42ba-89d7-66b598b77197","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698968514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698978213Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"session_id":"b8eaa205-d349-42ba-89d7-66b598b77197"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698988122Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Activated sub_volume 947b4a65-8ea8-41bc-bd01-3da0025fa536","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.698997081Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.699006179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.699360324Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"task":"crutest"} 01/30 Write block 491 len 4096 data: 6 02/30 Read block 116 len 4096 03/30 Write block 404 len 16384 data: 5 7 8 9 04/30 Write block 235 len 8192 data: 4 4 05/30 Read block 341 len 12288 06/30 Read block 515 len 32768 07/30 Read block 215 len 20480 08/30 Flush 09/30 Write block 592 len 12288 data: 9 9 8 10/30 Flush 11/30 Flush 12/30 Write block 45 len 20480 data: 9 9 10 10 10 13/30 Read block 412 len 8192 14/30 Read block 204 len 40960 15/30 Write block 42 len 16384 data: 7 7 8 10 16/30 Write block 85 len 16384 data: 6 6 6 6 17/30 Read block 324 len 8192 18/30 Read block 223 len 16384 19/30 Read block 107 len 4096 20/30 Flush 21/30 Write block 505 len 32768 data: 4 6 6 7 7 7 11 10 22/30 Flush 23/30 Write block 527 len 40960 data: 10 10 8 8 8 7 9 8 9 7 24/30 Read block 137 len 24576 25/30 Read block 444 len 28672 26/30 Write block 545 len 20480 data: 7 8 9 9 9 27/30 Write block 434 len 28672 data: 8 6 6 8 8 9 10 28/30 Write block 235 len 4096 data: 5 29/30 Write block 510 len 24576 data: 8 12 11 10 9 7 30/30 Read block 164 len 20480 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:33.708469925Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:33.708681068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1490,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 77 77 77 137 137 137 F F F 2 040-059 78 78 75 141 141 134 F F F <--- 3 060-079 74 74 74 133 133 133 F F F 4 080-099 78 78 63 141 141 109 F F F <--- 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 76 76 76 136 136 136 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 78 78 68 138 138 120 T T F <--- 12 240-259 77 77 77 137 137 137 F F F 13 260-279 77 77 77 137 137 137 F F F 14 280-299 77 77 77 137 137 137 F F F 15 300-319 77 77 77 137 137 137 F F F 16 320-339 74 74 74 133 133 133 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 77 77 77 137 137 137 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 78 78 77 138 138 137 F F T <--- 21 420-439 74 74 74 132 132 132 T T F <--- 22 440-459 74 74 74 132 132 132 T T F <--- 23 460-479 77 77 77 137 137 137 F F F 24 480-499 78 78 74 138 138 133 F F T <--- 25 500-519 78 78 75 142 142 134 T T F <--- 26 520-539 77 77 77 137 137 137 T T F <--- 27 540-559 66 66 66 118 118 118 T T F <--- 28 560-579 72 72 72 129 129 129 F F F 29 580-599 78 78 70 139 139 125 F F F <--- Max gen: 78, Max flush: 142 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 77 77 77 137 137 137 F F F 2 040-059 78 78 78 141 141 141 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 72 72 72 130 130 130 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 F F F 8 160-179 76 76 76 136 136 136 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 79 79 79 143 143 143 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 77 77 77 137 137 137 F F F 14 280-299 77 77 77 137 137 137 F F F 15 300-319 77 77 77 137 137 137 F F F 16 320-339 74 74 74 133 133 133 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 77 77 77 137 137 137 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 78 78 78 138 138 138 F F F 21 420-439 79 79 79 143 143 143 F F F 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 78 78 78 138 138 138 F F F 25 500-519 79 79 79 143 143 143 F F F 26 520-539 79 79 79 143 143 143 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 78 78 78 139 139 139 F F F Max gen: 79, Max flush: 143 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.987474756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.996311203Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.996625232Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.996658678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.996668416Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.996681035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Crucible c7468e42-c03a-40d4-a92d-704ff6d6c65d has session id: cd7c819b-57be-439a-b8f3-e7018070beae","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997091142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Upstairs opts: Upstairs UUID: c7468e42-c03a-40d4-a92d-704ff6d6c65d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997133816Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Crucible stats registered with UUID: c7468e42-c03a-40d4-a92d-704ff6d6c65d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997146175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:47.997156333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997294096Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997312813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997324012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997546343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997560631Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"c7468e42-c03a-40d4-a92d-704ff6d6c65d active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997614374Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997624723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"0","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997635421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"1","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.99764628Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"2","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997656849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997667787Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997678706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"ds_connection connected from Ok(127.0.0.1:42755)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997728629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"ds_connection connected from Ok(127.0.0.1:46793)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997760685Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"ds_connection connected from Ok(127.0.0.1:60829)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.997772164Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998476523Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"1","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.99849722Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"0","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998509878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"2","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998828417Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 141, 133, 141, 130, 130, 128, 136, 105, 124, 143]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998850484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[0]R generation[0..12]: [77, 77, 78, 74, 78, 72, 72, 71, 76, 60, 70, 79]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998862223Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998874541Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 141, 133, 141, 130, 130, 128, 136, 105, 124, 143]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.99888644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[1]R generation[0..12]: [77, 77, 78, 74, 78, 72, 72, 71, 76, 60, 70, 79]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998897978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998910296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 141, 133, 141, 130, 130, 128, 136, 105, 124, 143]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998922235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[2]R generation[0..12]: [77, 77, 78, 74, 78, 72, 72, 71, 76, 60, 70, 79]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998935813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998948022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.99895853Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998969609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998980217Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.998990896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"":"downstairs","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999002065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999012013Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"0","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999023062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"1","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.99903403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"client":"2","":"downstairs","session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999044979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999055108Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"c7468e42-c03a-40d4-a92d-704ff6d6c65d is now active with session: cd7c819b-57be-439a-b8f3-e7018070beae","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999066196Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999075995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"session_id":"cd7c819b-57be-439a-b8f3-e7018070beae"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999086284Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Activated sub_volume c7468e42-c03a-40d4-a92d-704ff6d6c65d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999095363Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999104481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:47.999416601Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"task":"crutest"} 01/30 Write block 488 len 40960 data: 4 4 6 7 6 6 6 6 5 4 02/30 Read block 20 len 20480 03/30 Write block 499 len 24576 data: 5 5 4 4 4 5 04/30 Read block 496 len 12288 05/30 Write block 250 len 16384 data: 6 6 6 8 06/30 Write block 402 len 16384 data: 7 8 6 8 07/30 Read block 524 len 16384 08/30 Write block 242 len 40960 data: 7 7 9 7 8 7 6 6 7 7 09/30 Flush 10/30 Flush 11/30 Write block 105 len 8192 data: 6 7 12/30 Write block 359 len 32768 data: 8 9 7 12 13 13 10 11 13/30 Read block 422 len 40960 14/30 Read block 140 len 4096 15/30 Write block 143 len 28672 data: 3 3 4 4 2 3 4 16/30 Write block 501 len 32768 data: 5 5 5 6 5 7 7 8 17/30 Read block 444 len 16384 18/30 Read block 30 len 8192 19/30 Read block 268 len 4096 20/30 Read block 390 len 28672 21/30 Read block 272 len 8192 22/30 Read block 213 len 40960 23/30 Read block 138 len 16384 24/30 Read block 25 len 24576 25/30 Write block 493 len 20480 data: 7 7 7 6 5 26/30 Write block 296 len 24576 data: 7 6 4 5 6 7 27/30 Read block 558 len 40960 28/30 Read block 310 len 36864 29/30 Read block 454 len 28672 30/30 Write block 271 len 8192 data: 7 6 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:55:48.009487118Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:55:48.009701401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1499,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 77 77 77 137 137 137 F F F 2 040-059 78 78 78 141 141 141 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 72 72 72 130 130 130 T F T <--- 6 120-139 72 72 72 130 130 130 F F F 7 140-159 71 71 71 128 128 128 T F T <--- 8 160-179 76 76 76 136 136 136 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 79 79 79 143 143 143 F F F 12 240-259 80 77 80 144 137 144 F F F <--- 13 260-279 77 77 77 137 137 137 T F T <--- 14 280-299 77 77 77 137 137 137 T F T <--- 15 300-319 77 77 77 137 137 137 T F T <--- 16 320-339 74 74 74 133 133 133 F F F 17 340-359 77 77 77 137 137 137 T F T <--- 18 360-379 77 77 77 137 137 137 T F T <--- 19 380-399 77 77 77 137 137 137 F F F 20 400-419 80 78 80 144 138 144 F F F <--- 21 420-439 79 79 79 143 143 143 F F F 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 80 78 80 144 138 144 T T T <--- 25 500-519 80 79 80 144 143 144 T F T <--- 26 520-539 79 79 79 143 143 143 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 78 78 78 139 139 139 F F F Max gen: 80, Max flush: 144 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 77 77 77 137 137 137 F F F 2 040-059 78 78 78 141 141 141 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 81 81 81 145 145 145 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 81 81 81 145 145 145 F F F 8 160-179 76 76 76 136 136 136 F F F 9 180-199 60 60 60 105 105 105 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 79 79 79 143 143 143 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 81 81 81 145 145 145 F F F 15 300-319 81 81 81 145 145 145 F F F 16 320-339 74 74 74 133 133 133 F F F 17 340-359 81 81 81 145 145 145 F F F 18 360-379 81 81 81 145 145 145 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 80 80 80 144 144 144 F F F 21 420-439 79 79 79 143 143 143 F F F 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 81 81 81 145 145 145 F F F 25 500-519 81 81 81 145 145 145 F F F 26 520-539 79 79 79 143 143 143 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 72 72 72 129 129 129 F F F 29 580-599 78 78 78 139 139 139 F F F Max gen: 81, Max flush: 145 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.287895306Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.296940756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297259245Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297293121Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297316518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Crucible 18acb54d-3e6e-42cb-b143-2ccb78402063 has session id: 86a11374-7203-4e01-90e4-b819c75d7366","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.29768662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Upstairs opts: Upstairs UUID: 18acb54d-3e6e-42cb-b143-2ccb78402063, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297704938Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Crucible stats registered with UUID: 18acb54d-3e6e-42cb-b143-2ccb78402063","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297716556Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:02.297727065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297891174Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297924899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.297939947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298104966Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298115425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"18acb54d-3e6e-42cb-b143-2ccb78402063 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.29815135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298162169Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"0","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298173127Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"1","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298184186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"2","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298195054Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298206463Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298217601Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"ds_connection connected from Ok(127.0.0.1:60572)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298286902Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"ds_connection connected from Ok(127.0.0.1:64603)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298447712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"ds_connection connected from Ok(127.0.0.1:34720)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.298475588Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299133703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"0","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.29915385Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"1","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299177087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"2","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299564997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 141, 133, 141, 145, 130, 145, 136, 105, 124, 143]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299594023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[0]R generation[0..12]: [77, 77, 78, 74, 78, 81, 72, 81, 76, 60, 70, 79]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299606732Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.29961952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 141, 133, 141, 145, 130, 145, 136, 105, 124, 143]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299631658Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[1]R generation[0..12]: [77, 77, 78, 74, 78, 81, 72, 81, 76, 60, 70, 79]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299643367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299655745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 141, 133, 141, 145, 130, 145, 136, 105, 124, 143]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299667744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[2]R generation[0..12]: [77, 77, 78, 74, 78, 81, 72, 81, 76, 60, 70, 79]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299679552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299691891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299702669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299713858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299724596Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299735235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"":"downstairs","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299748013Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299758052Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"0","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299769351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"1","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299780749Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"client":"2","":"downstairs","session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299792198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299802686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"18acb54d-3e6e-42cb-b143-2ccb78402063 is now active with session: 86a11374-7203-4e01-90e4-b819c75d7366","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299813945Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299824014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"session_id":"86a11374-7203-4e01-90e4-b819c75d7366"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299834382Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Activated sub_volume 18acb54d-3e6e-42cb-b143-2ccb78402063","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.299843661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.29985297Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.300250238Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"task":"crutest"} 01/30 Write block 107 len 16384 data: 7 6 5 5 02/30 Read block 85 len 28672 03/30 Read block 448 len 20480 04/30 Read block 445 len 8192 05/30 Flush 06/30 Write block 183 len 28672 data: 8 7 7 8 6 5 5 07/30 Write block 331 len 40960 data: 8 8 8 8 7 6 5 4 4 6 08/30 Read block 112 len 20480 09/30 Read block 533 len 8192 10/30 Write block 219 len 20480 data: 11 11 10 11 10 11/30 Read block 525 len 40960 12/30 Write block 174 len 20480 data: 6 7 8 8 7 13/30 Read block 507 len 16384 14/30 Write block 575 len 16384 data: 7 7 7 6 15/30 Flush 16/30 Write block 595 len 4096 data: 8 17/30 Write block 327 len 16384 data: 8 11 10 7 18/30 Flush 19/30 Write block 353 len 36864 data: 3 4 4 4 5 8 9 10 8 20/30 Read block 245 len 8192 21/30 Read block 380 len 16384 22/30 Write block 38 len 32768 data: 5 5 7 8 8 8 9 11 23/30 Read block 156 len 24576 24/30 Read block 106 len 16384 25/30 Read block 486 len 40960 26/30 Read block 51 len 40960 27/30 Write block 423 len 4096 data: 4 28/30 Write block 405 len 20480 data: 9 9 10 8 6 29/30 Read block 502 len 28672 30/30 Read block 161 len 28672 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 0 25 Done 25 0 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1004)) Some(JobId(1017)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:02.310658992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:02.310861106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1508,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 77 77 77 137 137 137 T F T <--- 2 040-059 78 78 78 141 141 141 T F T <--- 3 060-079 74 74 74 133 133 133 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 82 82 82 146 146 146 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 81 81 81 145 145 145 F F F 8 160-179 82 76 82 147 136 147 F F F <--- 9 180-199 82 60 82 147 105 147 F F F <--- 10 200-219 82 70 82 147 124 147 F F F <--- 11 220-239 82 79 82 147 143 147 F F F <--- 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 81 81 81 145 145 145 F F F 15 300-319 81 81 81 145 145 145 F F F 16 320-339 82 74 82 148 133 148 F F F <--- 17 340-359 82 81 82 147 145 147 T F T <--- 18 360-379 81 81 81 145 145 145 T F T <--- 19 380-399 77 77 77 137 137 137 F F F 20 400-419 80 80 80 144 144 144 T F T <--- 21 420-439 79 79 79 143 143 143 T F T <--- 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 81 81 81 145 145 145 F F F 25 500-519 81 81 81 145 145 145 F F F 26 520-539 79 79 79 143 143 143 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 82 72 82 147 129 147 F F F <--- 29 580-599 82 78 82 148 139 148 F F F <--- Max gen: 82, Max flush: 148 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 83 83 83 149 149 149 F F F 2 040-059 83 83 83 149 149 149 F F F 3 060-079 74 74 74 133 133 133 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 82 82 82 146 146 146 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 81 81 81 145 145 145 F F F 8 160-179 82 82 82 147 147 147 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 82 82 82 147 147 147 F F F 11 220-239 82 82 82 147 147 147 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 81 81 81 145 145 145 F F F 15 300-319 81 81 81 145 145 145 F F F 16 320-339 82 82 82 148 148 148 F F F 17 340-359 83 83 83 149 149 149 F F F 18 360-379 83 83 83 149 149 149 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 83 83 83 149 149 149 F F F 21 420-439 83 83 83 149 149 149 F F F 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 81 81 81 145 145 145 F F F 25 500-519 81 81 81 145 145 145 F F F 26 520-539 79 79 79 143 143 143 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 82 82 82 147 147 147 F F F 29 580-599 82 82 82 148 148 148 F F F Max gen: 83, Max flush: 149 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.592697881Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.601951664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602259614Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602293829Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602303388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602318566Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Crucible c85274c7-02f0-4539-9b19-46e830c25e3b has session id: 027e5884-c4c6-4e3c-adf8-4f3cbe450ad3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602728613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Upstairs opts: Upstairs UUID: c85274c7-02f0-4539-9b19-46e830c25e3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602774577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Crucible stats registered with UUID: c85274c7-02f0-4539-9b19-46e830c25e3b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602786156Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:16.602796354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602936346Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602968912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.602980911Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.60314191Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603160327Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"c85274c7-02f0-4539-9b19-46e830c25e3b active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603189474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603199792Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"0","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603210391Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"1","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603220889Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"2","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603231388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603245066Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603255875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45287)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603324816Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54138)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603352233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50207)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.603364821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604052372Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"2","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604077849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"0","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604116484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"1","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604419195Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[0]R flush_numbers[0..12]: [137, 149, 149, 133, 141, 146, 130, 145, 147, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604440802Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[0]R generation[0..12]: [77, 83, 83, 74, 78, 82, 72, 81, 82, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.60445264Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604465029Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[1]R flush_numbers[0..12]: [137, 149, 149, 133, 141, 146, 130, 145, 147, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604476767Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[1]R generation[0..12]: [77, 83, 83, 74, 78, 82, 72, 81, 82, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604488346Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604503144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[2]R flush_numbers[0..12]: [137, 149, 149, 133, 141, 146, 130, 145, 147, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604514882Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[2]R generation[0..12]: [77, 83, 83, 74, 78, 82, 72, 81, 82, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604526281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604538389Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604548838Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604559766Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604570045Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604580394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"":"downstairs","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604591362Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604601161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"0","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.60461208Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"1","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604622718Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"client":"2","":"downstairs","session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604633507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604643535Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"c85274c7-02f0-4539-9b19-46e830c25e3b is now active with session: 027e5884-c4c6-4e3c-adf8-4f3cbe450ad3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604659933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604669652Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"session_id":"027e5884-c4c6-4e3c-adf8-4f3cbe450ad3"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604681091Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Activated sub_volume c85274c7-02f0-4539-9b19-46e830c25e3b","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604689989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.604698928Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.605068561Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"task":"crutest"} 01/30 Write block 564 len 40960 data: 9 9 9 7 6 6 6 6 6 8 02/30 Read block 575 len 4096 03/30 Read block 354 len 40960 04/30 Read block 386 len 40960 05/30 Read block 79 len 20480 06/30 Read block 480 len 40960 07/30 Write block 225 len 16384 data: 12 10 10 8 08/30 Read block 298 len 36864 09/30 Write block 341 len 8192 data: 6 4 10/30 Read block 343 len 16384 11/30 Read block 218 len 4096 12/30 Read block 220 len 8192 13/30 Flush 14/30 Write block 432 len 4096 data: 7 15/30 Write block 339 len 20480 data: 5 7 7 5 5 16/30 Write block 323 len 24576 data: 5 4 5 7 9 12 17/30 Read block 333 len 12288 18/30 Write block 520 len 12288 data: 11 11 11 19/30 Read block 305 len 8192 20/30 Flush 21/30 Write block 427 len 20480 data: 5 4 5 6 8 22/30 Read block 217 len 40960 23/30 Write block 347 len 32768 data: 5 3 4 7 5 3 4 5 24/30 Flush 25/30 Write block 349 len 32768 data: 5 8 6 4 5 6 5 5 26/30 Flush 27/30 Write block 26 len 20480 data: 7 6 5 6 3 28/30 Write block 74 len 28672 data: 3 3 2 2 2 2 2 29/30 Read block 401 len 36864 30/30 Read block 61 len 8192 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:16.614823349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:16.615024953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1517,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 83 83 83 149 149 149 T F T <--- 2 040-059 83 83 83 149 149 149 F F F 3 060-079 74 74 74 133 133 133 T F T <--- 4 080-099 78 78 78 141 141 141 T F T <--- 5 100-119 82 82 82 146 146 146 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 81 81 81 145 145 145 F F F 8 160-179 82 82 82 147 147 147 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 82 82 82 147 147 147 F F F 11 220-239 84 82 84 150 147 150 F F F <--- 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 81 81 81 145 145 145 F F F 15 300-319 81 81 81 145 145 145 F F F 16 320-339 84 82 84 151 148 151 F F F <--- 17 340-359 84 83 84 153 149 153 F F F <--- 18 360-379 83 83 83 149 149 149 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 83 83 83 149 149 149 F F F 21 420-439 84 83 84 152 149 152 F F F <--- 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 81 81 81 145 145 145 F F F 25 500-519 81 81 81 145 145 145 F F F 26 520-539 84 79 84 151 143 151 F F F <--- 27 540-559 79 79 79 143 143 143 F F F 28 560-579 84 82 84 150 147 150 F T F <--- 29 580-599 82 82 82 148 148 148 F F F Max gen: 84, Max flush: 153 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 85 85 85 154 154 154 F F F 2 040-059 83 83 83 149 149 149 F F F 3 060-079 85 85 85 154 154 154 F F F 4 080-099 85 85 85 154 154 154 F F F 5 100-119 82 82 82 146 146 146 F F F 6 120-139 72 72 72 130 130 130 F F F 7 140-159 81 81 81 145 145 145 F F F 8 160-179 82 82 82 147 147 147 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 82 82 82 147 147 147 F F F 11 220-239 84 84 84 150 150 150 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 81 81 81 145 145 145 F F F 15 300-319 81 81 81 145 145 145 F F F 16 320-339 84 84 84 151 151 151 F F F 17 340-359 84 84 84 153 153 153 F F F 18 360-379 83 83 83 149 149 149 F F F 19 380-399 77 77 77 137 137 137 F F F 20 400-419 83 83 83 149 149 149 F F F 21 420-439 84 84 84 152 152 152 F F F 22 440-459 79 79 79 143 143 143 F F F 23 460-479 77 77 77 137 137 137 F F F 24 480-499 81 81 81 145 145 145 F F F 25 500-519 81 81 81 145 145 145 F F F 26 520-539 84 84 84 151 151 151 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 82 82 82 148 148 148 F F F Max gen: 85, Max flush: 154 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.889782282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:30.897460169Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.897510343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.904915945Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905225535Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905271819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905281508Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905294296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Crucible 1662fcbc-bac6-4cee-aa75-4fe9bd593ab5 has session id: 48d8d792-1783-4cc9-84b3-d3b151516631","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905658839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Upstairs opts: Upstairs UUID: 1662fcbc-bac6-4cee-aa75-4fe9bd593ab5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905705563Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Crucible stats registered with UUID: 1662fcbc-bac6-4cee-aa75-4fe9bd593ab5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905720901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:30.905733109Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905873811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905891419Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.905902217Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906080194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906090433Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"1662fcbc-bac6-4cee-aa75-4fe9bd593ab5 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906107871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906117269Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"0","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906127798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"1","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906138367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"2","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906148945Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.90618709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906211557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"ds_connection connected from Ok(127.0.0.1:53055)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906224126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"ds_connection connected from Ok(127.0.0.1:64291)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906235554Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"ds_connection connected from Ok(127.0.0.1:63590)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.906246883Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907028132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"1","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907059578Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"0","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907083395Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"2","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907442858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[0]R flush_numbers[0..12]: [137, 154, 149, 154, 154, 146, 130, 145, 147, 147, 147, 150]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907468635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[0]R generation[0..12]: [77, 85, 83, 85, 85, 82, 72, 81, 82, 82, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907480543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907492672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[1]R flush_numbers[0..12]: [137, 154, 149, 154, 154, 146, 130, 145, 147, 147, 147, 150]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.90750428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[1]R generation[0..12]: [77, 85, 83, 85, 85, 82, 72, 81, 82, 82, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907515849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907528117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[2]R flush_numbers[0..12]: [137, 154, 149, 154, 154, 146, 130, 145, 147, 147, 147, 150]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907539786Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[2]R generation[0..12]: [77, 85, 83, 85, 85, 82, 72, 81, 82, 82, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907551164Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907562943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907575841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907586609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907596808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907606997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"":"downstairs","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907617685Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907627214Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"0","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907637873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"1","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907648341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"client":"2","":"downstairs","session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.90765914Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907669059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"1662fcbc-bac6-4cee-aa75-4fe9bd593ab5 is now active with session: 48d8d792-1783-4cc9-84b3-d3b151516631","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907679927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907689446Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"session_id":"48d8d792-1783-4cc9-84b3-d3b151516631"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907699365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Activated sub_volume 1662fcbc-bac6-4cee-aa75-4fe9bd593ab5","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907708444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.907717383Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.908072677Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} 01/30 Write block 285 len 36864 data: 6 9 9 11 9 8 9 8 8 02/30 Read block 283 len 20480 03/30 Write block 564 len 12288 data: 10 10 10 04/30 Write block 525 len 32768 data: 7 9 11 11 9 9 9 8 05/30 Read block 382 len 20480 06/30 Read block 156 len 36864 07/30 Write block 88 len 40960 data: 7 4 5 4 5 6 7 7 8 8 08/30 Write block 75 len 36864 data: 4 3 3 3 3 3 2 3 3 09/30 Read block 528 len 28672 10/30 Write block 449 len 24576 data: 6 6 4 4 4 5 11/30 Write block 186 len 28672 data: 9 7 6 6 5 7 9 12/30 Flush 13/30 Write block 121 len 20480 data: 8 8 8 7 6 14/30 Read block 440 len 12288 15/30 Read block 244 len 8192 16/30 Read block 260 len 28672 17/30 Read block 394 len 28672 18/30 Write block 82 len 4096 data: 4 19/30 Write block 482 len 20480 data: 7 7 6 6 5 20/30 Write block 509 len 24576 data: 8 9 13 12 11 10 21/30 Write block 490 len 8192 data: 7 8 22/30 Write block 593 len 24576 data: 10 9 9 7 5 5 23/30 Write block 381 len 28672 data: 9 10 7 7 8 9 8 24/30 Write block 311 len 8192 data: 14 10 25/30 Write block 475 len 40960 data: 8 7 6 6 7 6 7 8 8 7 26/30 Write block 335 len 12288 data: 8 7 6 27/30 Read block 62 len 12288 28/30 Write block 157 len 16384 data: 7 8 9 8 29/30 Read block 414 len 40960 30/30 Read block 442 len 24576 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:30.917085321Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:30.917284745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1526,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 85 85 85 154 154 154 F F F 2 040-059 83 83 83 149 149 149 F F F 3 060-079 85 86 86 154 155 155 F F F <--- 4 080-099 85 86 86 154 155 155 F T T <--- 5 100-119 82 82 82 146 146 146 F F F 6 120-139 72 72 72 130 130 130 F T T <--- 7 140-159 81 81 81 145 145 145 F T T <--- 8 160-179 82 82 82 147 147 147 F T T <--- 9 180-199 82 86 86 147 155 155 F F F <--- 10 200-219 82 82 82 147 147 147 F F F 11 220-239 84 84 84 150 150 150 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 81 86 86 145 155 155 F F F <--- 15 300-319 81 81 81 145 145 145 F T T <--- 16 320-339 84 84 84 151 151 151 F T T <--- 17 340-359 84 84 84 153 153 153 F F F 18 360-379 83 83 83 149 149 149 F F F 19 380-399 77 77 77 137 137 137 F T T <--- 20 400-419 83 83 83 149 149 149 F F F 21 420-439 84 84 84 152 152 152 F F F 22 440-459 79 86 86 143 155 155 F F F <--- 23 460-479 77 77 77 137 137 137 F T T <--- 24 480-499 81 81 81 145 145 145 F T T <--- 25 500-519 81 81 81 145 145 145 F T T <--- 26 520-539 84 86 86 151 155 155 F F F <--- 27 540-559 79 79 79 143 143 143 F F F 28 560-579 84 86 86 150 155 155 F F F <--- 29 580-599 82 82 82 148 148 148 F T T <--- Max gen: 86, Max flush: 155 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 85 85 85 154 154 154 F F F 2 040-059 83 83 83 149 149 149 F F F 3 060-079 86 86 86 155 155 155 F F F 4 080-099 87 87 87 156 156 156 F F F 5 100-119 82 82 82 146 146 146 F F F 6 120-139 87 87 87 156 156 156 F F F 7 140-159 87 87 87 156 156 156 F F F 8 160-179 87 87 87 156 156 156 F F F 9 180-199 86 86 86 155 155 155 F F F 10 200-219 82 82 82 147 147 147 F F F 11 220-239 84 84 84 150 150 150 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 81 81 81 145 145 145 F F F 14 280-299 86 86 86 155 155 155 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 84 84 84 153 153 153 F F F 18 360-379 83 83 83 149 149 149 F F F 19 380-399 87 87 87 156 156 156 F F F 20 400-419 83 83 83 149 149 149 F F F 21 420-439 84 84 84 152 152 152 F F F 22 440-459 86 86 86 155 155 155 F F F 23 460-479 87 87 87 156 156 156 F F F 24 480-499 87 87 87 156 156 156 F F F 25 500-519 87 87 87 156 156 156 F F F 26 520-539 86 86 86 155 155 155 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 86 86 86 155 155 155 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 87, Max flush: 156 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.204356433Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:45.211835575Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.211889758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.218928288Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.21921746Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219248746Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219257415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219269164Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Crucible 4abd703e-262a-4c34-8de5-5158fffc673e has session id: 639b7a8a-fd2c-4724-ad83-a5cb10fef1e0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219617939Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Upstairs opts: Upstairs UUID: 4abd703e-262a-4c34-8de5-5158fffc673e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219647055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Crucible stats registered with UUID: 4abd703e-262a-4c34-8de5-5158fffc673e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219658583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:45.219668052Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219778988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219810894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219824362Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219963634Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219975872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"4abd703e-262a-4c34-8de5-5158fffc673e active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.219999709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220009468Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"0","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220019327Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"1","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220029245Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"2","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220038874Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220049043Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220059062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:60016)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220113324Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48904)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220128732Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58679)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220182236Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.22076912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"1","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220785248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"2","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.220890744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"0","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221165188Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[0]R flush_numbers[0..12]: [137, 154, 149, 155, 156, 146, 156, 156, 156, 155, 147, 150]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221199334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[0]R generation[0..12]: [77, 85, 83, 86, 87, 82, 87, 87, 87, 86, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221210453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221221531Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[1]R flush_numbers[0..12]: [137, 154, 149, 155, 156, 146, 156, 156, 156, 155, 147, 150]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.22123228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[1]R generation[0..12]: [77, 85, 83, 86, 87, 82, 87, 87, 87, 86, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221242898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221253997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[2]R flush_numbers[0..12]: [137, 154, 149, 155, 156, 146, 156, 156, 156, 155, 147, 150]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221264726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[2]R generation[0..12]: [77, 85, 83, 86, 87, 82, 87, 87, 87, 86, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221275214Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221286283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221296022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.22130605Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221315549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221325248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"":"downstairs","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221336806Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221345745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"0","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221355624Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"1","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221365443Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"client":"2","":"downstairs","session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221375511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.22138479Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"4abd703e-262a-4c34-8de5-5158fffc673e is now active with session: 639b7a8a-fd2c-4724-ad83-a5cb10fef1e0","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221394889Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221403958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"session_id":"639b7a8a-fd2c-4724-ad83-a5cb10fef1e0"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221413106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Activated sub_volume 4abd703e-262a-4c34-8de5-5158fffc673e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221421265Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221429624Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.221735275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} 01/30 Write block 175 len 40960 data: 8 9 9 8 4 6 7 7 9 8 02/30 Read block 387 len 20480 03/30 Write block 179 len 32768 data: 5 7 8 8 10 9 8 10 04/30 Read block 17 len 24576 05/30 Read block 397 len 8192 06/30 Write block 464 len 40960 data: 9 9 7 9 10 10 10 12 13 11 07/30 Read block 539 len 8192 08/30 Write block 222 len 8192 data: 12 11 09/30 Write block 440 len 24576 data: 11 10 12 10 9 9 10/30 Write block 235 len 20480 data: 6 5 5 6 5 11/30 Read block 45 len 8192 12/30 Flush 13/30 Write block 79 len 12288 data: 4 4 3 14/30 Read block 146 len 12288 15/30 Read block 402 len 12288 16/30 Write block 95 len 32768 data: 8 9 9 8 8 6 5 4 17/30 Write block 108 len 16384 data: 7 6 6 4 18/30 Write block 237 len 28672 data: 6 7 6 4 6 8 8 19/30 Flush 20/30 Write block 288 len 16384 data: 12 10 9 10 21/30 Write block 516 len 20480 data: 8 12 9 10 12 22/30 Write block 448 len 36864 data: 7 7 7 5 5 5 6 3 3 23/30 Write block 275 len 16384 data: 9 8 6 4 24/30 Flush 25/30 Write block 362 len 36864 data: 13 14 14 11 12 11 10 11 9 26/30 Write block 108 len 40960 data: 8 7 7 5 5 6 6 7 4 5 27/30 Write block 113 len 36864 data: 7 7 8 5 6 5 6 6 9 28/30 Read block 71 len 20480 29/30 Write block 276 len 24576 data: 9 7 5 4 5 4 30/30 Read block 474 len 16384 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 6 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 2 30 Done 2 30 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:45.231232646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:45.231427161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1535,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 85 85 85 154 154 154 F F F 2 040-059 83 83 83 149 149 149 F F F 3 060-079 86 88 88 155 158 158 F F F <--- 4 080-099 87 88 88 156 158 158 F F F <--- 5 100-119 82 88 88 146 158 158 F T T <--- 6 120-139 87 87 87 156 156 156 F T T <--- 7 140-159 87 87 87 156 156 156 F F F 8 160-179 87 88 88 156 157 157 T F F <--- 9 180-199 86 88 88 155 157 157 T F F <--- 10 200-219 82 82 82 147 147 147 F F F 11 220-239 84 88 88 150 158 158 F F F <--- 12 240-259 80 88 88 144 158 158 F F F <--- 13 260-279 81 88 88 145 159 159 F T T <--- 14 280-299 86 88 88 155 159 159 F T T <--- 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 84 84 84 153 153 153 F F F 18 360-379 83 83 83 149 149 149 F T T <--- 19 380-399 87 87 87 156 156 156 F F F 20 400-419 83 83 83 149 149 149 F F F 21 420-439 84 84 84 152 152 152 F F F 22 440-459 86 88 88 155 159 159 F F F <--- 23 460-479 87 88 88 156 157 157 F F F <--- 24 480-499 87 87 87 156 156 156 F F F 25 500-519 87 88 88 156 159 159 F F F <--- 26 520-539 86 88 88 155 159 159 F F F <--- 27 540-559 79 79 79 143 143 143 F F F 28 560-579 86 86 86 155 155 155 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 88, Max flush: 159 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 F F F 1 020-039 85 85 85 154 154 154 F F F 2 040-059 83 83 83 149 149 149 F F F 3 060-079 88 88 88 158 158 158 F F F 4 080-099 88 88 88 158 158 158 F F F 5 100-119 89 89 89 160 160 160 F F F 6 120-139 89 89 89 160 160 160 F F F 7 140-159 87 87 87 156 156 156 F F F 8 160-179 88 88 88 157 157 157 F F F 9 180-199 88 88 88 157 157 157 F F F 10 200-219 82 82 82 147 147 147 F F F 11 220-239 88 88 88 158 158 158 F F F 12 240-259 88 88 88 158 158 158 F F F 13 260-279 89 89 89 160 160 160 F F F 14 280-299 89 89 89 160 160 160 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 84 84 84 153 153 153 F F F 18 360-379 89 89 89 160 160 160 F F F 19 380-399 87 87 87 156 156 156 F F F 20 400-419 83 83 83 149 149 149 F F F 21 420-439 84 84 84 152 152 152 F F F 22 440-459 88 88 88 159 159 159 F F F 23 460-479 88 88 88 157 157 157 F F F 24 480-499 87 87 87 156 156 156 F F F 25 500-519 88 88 88 159 159 159 F F F 26 520-539 88 88 88 159 159 159 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 86 86 86 155 155 155 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 89, Max flush: 160 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.512433953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.521302126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.521645592Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.521694195Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.521706484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.521719262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Crucible 2e9d7b7b-5afc-4e6b-b9f0-99eda64327b6 has session id: 3d0e4f20-62bf-475d-b4a9-d4030f06ff0a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522137338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Upstairs opts: Upstairs UUID: 2e9d7b7b-5afc-4e6b-b9f0-99eda64327b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522156456Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Crucible stats registered with UUID: 2e9d7b7b-5afc-4e6b-b9f0-99eda64327b6","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522171454Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:59.522181512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.52235397Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522386516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522398874Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522543725Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522557804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"2e9d7b7b-5afc-4e6b-b9f0-99eda64327b6 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522597069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522607537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"0","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522618296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"1","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522631124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"2","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522641603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522672729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522695256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61066)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522768646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43370)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522780975Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:32798)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.522792393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523547256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"1","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523565023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"0","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523607488Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"2","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523925047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[0]R flush_numbers[0..12]: [137, 154, 149, 158, 158, 160, 160, 156, 157, 157, 147, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523950484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[0]R generation[0..12]: [77, 85, 83, 88, 88, 89, 89, 87, 88, 88, 82, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523962932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.52397548Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[1]R flush_numbers[0..12]: [137, 154, 149, 158, 158, 160, 160, 156, 157, 157, 147, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.523987559Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[1]R generation[0..12]: [77, 85, 83, 88, 88, 89, 89, 87, 88, 88, 82, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524016285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524028623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[2]R flush_numbers[0..12]: [137, 154, 149, 158, 158, 160, 160, 156, 157, 157, 147, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524040432Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[2]R generation[0..12]: [77, 85, 83, 88, 88, 89, 89, 87, 88, 88, 82, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.52405187Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524063889Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524074248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524085026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524095315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524105683Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"":"downstairs","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524116872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524126661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"0","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524137569Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"1","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524148558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"client":"2","":"downstairs","session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524159696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524169795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"2e9d7b7b-5afc-4e6b-b9f0-99eda64327b6 is now active with session: 3d0e4f20-62bf-475d-b4a9-d4030f06ff0a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524181774Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524191342Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"session_id":"3d0e4f20-62bf-475d-b4a9-d4030f06ff0a"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524201351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Activated sub_volume 2e9d7b7b-5afc-4e6b-b9f0-99eda64327b6","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.52421019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524219189Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.524592061Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"task":"crutest"} 01/30 Flush 02/30 Write block 202 len 16384 data: 5 5 7 6 03/30 Read block 199 len 20480 04/30 Read block 466 len 28672 05/30 Read block 553 len 28672 06/30 Read block 45 len 16384 07/30 Write block 123 len 36864 data: 9 8 7 5 3 3 4 4 4 08/30 Write block 96 len 16384 data: 10 10 9 9 09/30 Flush 10/30 Write block 99 len 40960 data: 10 7 6 5 5 6 7 8 8 9 11/30 Write block 207 len 4096 data: 7 12/30 Read block 191 len 40960 13/30 Write block 461 len 16384 data: 8 7 7 10 14/30 Write block 149 len 28672 data: 5 4 4 3 3 5 4 15/30 Write block 564 len 28672 data: 11 11 11 8 7 7 7 16/30 Read block 331 len 24576 17/30 Read block 267 len 16384 18/30 Write block 405 len 20480 data: 10 10 11 9 7 19/30 Read block 440 len 36864 20/30 Write block 433 len 24576 data: 7 9 7 7 9 9 21/30 Write block 35 len 40960 data: 7 7 5 6 6 8 9 9 9 10 22/30 Write block 36 len 8192 data: 8 6 23/30 Read block 197 len 24576 24/30 Read block 411 len 28672 25/30 Read block 394 len 36864 26/30 Write block 348 len 8192 data: 4 6 27/30 Read block 482 len 40960 28/30 Read block 79 len 36864 29/30 Read block 579 len 28672 30/30 Write block 11 len 8192 data: 7 5 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 19 21 Done 20 20 2 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:56:59.533730409Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:56:59.533930283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1544,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 137 137 137 T T F <--- 1 020-039 85 85 85 154 154 154 T T F <--- 2 040-059 83 83 83 149 149 149 T T F <--- 3 060-079 88 88 88 158 158 158 F F F 4 080-099 90 90 90 162 162 162 T T T 5 100-119 89 89 89 160 160 160 T T T 6 120-139 90 90 90 162 162 162 F F F 7 140-159 87 87 87 156 156 156 T T F <--- 8 160-179 88 88 88 157 157 157 F F F 9 180-199 88 88 88 157 157 157 F F F 10 200-219 90 90 90 162 162 162 T T T 11 220-239 88 88 88 158 158 158 F F F 12 240-259 88 88 88 158 158 158 F F F 13 260-279 89 89 89 160 160 160 F F F 14 280-299 89 89 89 160 160 160 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 84 84 84 153 153 153 T T F <--- 18 360-379 89 89 89 160 160 160 F F F 19 380-399 87 87 87 156 156 156 F F F 20 400-419 83 83 83 149 149 149 T T F <--- 21 420-439 84 84 84 152 152 152 T T F <--- 22 440-459 88 88 88 159 159 159 F F F 23 460-479 88 88 88 157 157 157 T T F <--- 24 480-499 87 87 87 156 156 156 F F F 25 500-519 88 88 88 159 159 159 F F F 26 520-539 88 88 88 159 159 159 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 86 86 86 155 155 155 T T F <--- 29 580-599 87 87 87 156 156 156 F F F Max gen: 90, Max flush: 162 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 163 163 163 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 91 91 91 163 163 163 F F F 3 060-079 88 88 88 158 158 158 F F F 4 080-099 91 91 91 163 163 163 F F F 5 100-119 91 91 91 163 163 163 F F F 6 120-139 90 90 90 162 162 162 F F F 7 140-159 91 91 91 163 163 163 F F F 8 160-179 88 88 88 157 157 157 F F F 9 180-199 88 88 88 157 157 157 F F F 10 200-219 91 91 91 163 163 163 F F F 11 220-239 88 88 88 158 158 158 F F F 12 240-259 88 88 88 158 158 158 F F F 13 260-279 89 89 89 160 160 160 F F F 14 280-299 89 89 89 160 160 160 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 91 91 91 163 163 163 F F F 18 360-379 89 89 89 160 160 160 F F F 19 380-399 87 87 87 156 156 156 F F F 20 400-419 91 91 91 163 163 163 F F F 21 420-439 91 91 91 163 163 163 F F F 22 440-459 88 88 88 159 159 159 F F F 23 460-479 91 91 91 163 163 163 F F F 24 480-499 87 87 87 156 156 156 F F F 25 500-519 88 88 88 159 159 159 F F F 26 520-539 88 88 88 159 159 159 F F F 27 540-559 79 79 79 143 143 143 F F F 28 560-579 91 91 91 163 163 163 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 91, Max flush: 163 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.803019936Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:13.810438316Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.810493669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.817555415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.81783072Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.817862736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.817871794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.817883813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Crucible 986c7b47-c87f-4f79-beed-1bbfa921e74e has session id: d3ba00e8-190f-41ad-b753-142dafcac9cb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818275142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Upstairs opts: Upstairs UUID: 986c7b47-c87f-4f79-beed-1bbfa921e74e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.81829391Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Crucible stats registered with UUID: 986c7b47-c87f-4f79-beed-1bbfa921e74e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818306248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:13.818315777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818473817Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818508232Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818519811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818639655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818657093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"986c7b47-c87f-4f79-beed-1bbfa921e74e active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818686169Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818695908Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"0","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818706116Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"1","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818716095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"2","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818727114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818737672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818748101Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:47509)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818793765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:44347)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818805004Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58859)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.818921599Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819463449Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"2","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819489785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"1","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819502074Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"0","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.81983986Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[0]R flush_numbers[0..12]: [163, 163, 163, 158, 163, 163, 162, 163, 157, 157, 163, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819866876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[0]R generation[0..12]: [91, 91, 91, 88, 91, 91, 90, 91, 88, 88, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819878635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819890093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[1]R flush_numbers[0..12]: [163, 163, 163, 158, 163, 163, 162, 163, 157, 157, 163, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819901152Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[1]R generation[0..12]: [91, 91, 91, 88, 91, 91, 90, 91, 88, 88, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819912031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819923349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[2]R flush_numbers[0..12]: [163, 163, 163, 158, 163, 163, 162, 163, 157, 157, 163, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819934388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[2]R generation[0..12]: [91, 91, 91, 88, 91, 91, 90, 91, 88, 88, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819945266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819956725Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819969363Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819979532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819989501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.819999109Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"":"downstairs","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820009358Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820018617Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"0","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820028665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"1","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820038674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"client":"2","":"downstairs","session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820048813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820058332Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 986c7b47-c87f-4f79-beed-1bbfa921e74e is now active with session: d3ba00e8-190f-41ad-b753-142dafcac9cb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.82007274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820081949Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"session_id":"d3ba00e8-190f-41ad-b753-142dafcac9cb"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820091527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Activated sub_volume 986c7b47-c87f-4f79-beed-1bbfa921e74e","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820099976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820108455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.820333296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} 01/30 Write block 103 len 20480 data: 6 7 8 9 9 02/30 Write block 73 len 4096 data: 3 03/30 Read block 431 len 8192 04/30 Write block 84 len 4096 data: 5 05/30 Read block 134 len 32768 06/30 Write block 11 len 28672 data: 8 6 5 5 6 6 6 07/30 Read block 534 len 20480 08/30 Write block 448 len 40960 data: 8 8 8 6 6 6 7 4 4 5 09/30 Read block 148 len 32768 10/30 Write block 266 len 16384 data: 5 6 6 5 11/30 Read block 185 len 8192 12/30 Write block 537 len 28672 data: 8 10 8 10 11 12 8 13/30 Read block 237 len 36864 14/30 Write block 552 len 20480 data: 6 7 8 7 7 15/30 Flush 16/30 Write block 154 len 20480 data: 6 5 6 8 9 17/30 Read block 317 len 12288 18/30 Write block 254 len 8192 data: 5 4 19/30 Flush 20/30 Write block 364 len 36864 data: 15 12 13 12 11 12 10 7 7 21/30 Read block 124 len 12288 22/30 Read block 462 len 32768 23/30 Write block 183 len 40960 data: 11 10 9 11 8 7 7 6 8 10 24/30 Read block 136 len 4096 25/30 Read block 62 len 32768 26/30 Read block 377 len 24576 27/30 Write block 158 len 20480 data: 10 10 9 8 10 28/30 Write block 280 len 4096 data: 6 29/30 Read block 120 len 4096 30/30 Write block 50 len 40960 data: 8 10 10 7 5 5 3 4 4 5 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Sent Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:13.829771335Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:13.829970929Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1553,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 92 92 163 164 164 F F F <--- 1 020-039 91 91 91 163 163 163 F F F 2 040-059 91 91 91 163 163 163 F T T <--- 3 060-079 88 92 92 158 164 164 F F F <--- 4 080-099 91 92 92 163 164 164 F F F <--- 5 100-119 91 92 92 163 164 164 F F F <--- 6 120-139 90 90 90 162 162 162 F F F 7 140-159 91 92 92 163 165 165 F T T <--- 8 160-179 88 88 88 157 157 157 F T T <--- 9 180-199 88 88 88 157 157 157 F T T <--- 10 200-219 91 91 91 163 163 163 F F F 11 220-239 88 88 88 158 158 158 F F F 12 240-259 88 92 92 158 165 165 F F F <--- 13 260-279 89 92 92 160 164 164 F F F <--- 14 280-299 89 89 89 160 160 160 F T T <--- 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 91 91 91 163 163 163 F F F 18 360-379 89 89 89 160 160 160 F T T <--- 19 380-399 87 87 87 156 156 156 F F F 20 400-419 91 91 91 163 163 163 F F F 21 420-439 91 91 91 163 163 163 F F F 22 440-459 88 92 92 159 164 164 F F F <--- 23 460-479 91 91 91 163 163 163 F F F 24 480-499 87 87 87 156 156 156 F F F 25 500-519 88 88 88 159 159 159 F F F 26 520-539 88 92 92 159 164 164 F F F <--- 27 540-559 79 92 92 143 164 164 F F F <--- 28 560-579 91 91 91 163 163 163 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 92, Max flush: 165 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 93 93 93 166 166 166 F F F 3 060-079 92 92 92 164 164 164 F F F 4 080-099 92 92 92 164 164 164 F F F 5 100-119 92 92 92 164 164 164 F F F 6 120-139 90 90 90 162 162 162 F F F 7 140-159 93 93 93 166 166 166 F F F 8 160-179 93 93 93 166 166 166 F F F 9 180-199 93 93 93 166 166 166 F F F 10 200-219 91 91 91 163 163 163 F F F 11 220-239 88 88 88 158 158 158 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 92 92 92 164 164 164 F F F 14 280-299 93 93 93 166 166 166 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 F F F 17 340-359 91 91 91 163 163 163 F F F 18 360-379 93 93 93 166 166 166 F F F 19 380-399 87 87 87 156 156 156 F F F 20 400-419 91 91 91 163 163 163 F F F 21 420-439 91 91 91 163 163 163 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 91 91 91 163 163 163 F F F 24 480-499 87 87 87 156 156 156 F F F 25 500-519 88 88 88 159 159 159 F F F 26 520-539 92 92 92 164 164 164 F F F 27 540-559 92 92 92 164 164 164 F F F 28 560-579 91 91 91 163 163 163 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 93, Max flush: 166 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.127860088Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136153765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136448637Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136476383Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136485002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136496481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Crucible 9d05fa42-991b-4fe8-8f76-cdf799ef3aeb has session id: 1e2ae5a7-51be-4c65-ab11-c2f2004d8126","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136838666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Upstairs opts: Upstairs UUID: 9d05fa42-991b-4fe8-8f76-cdf799ef3aeb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136905578Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Crucible stats registered with UUID: 9d05fa42-991b-4fe8-8f76-cdf799ef3aeb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.136915896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:28.136925225Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137052959Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137083095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137094413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137253993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137268691Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"9d05fa42-991b-4fe8-8f76-cdf799ef3aeb active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137306986Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137316335Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"0","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137326083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"1","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137335752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"2","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137345521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.13735549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137365558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"ds_connection connected from Ok(127.0.0.1:42479)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137382646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"ds_connection connected from Ok(127.0.0.1:39284)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137392835Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"ds_connection connected from Ok(127.0.0.1:49787)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.137487063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138089145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"0","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138194341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"2","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138230736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"1","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138486543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[0]R flush_numbers[0..12]: [164, 163, 166, 164, 164, 164, 162, 166, 166, 166, 163, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138508201Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[0]R generation[0..12]: [92, 91, 93, 92, 92, 92, 90, 93, 93, 93, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138518789Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138529858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[1]R flush_numbers[0..12]: [164, 163, 166, 164, 164, 164, 162, 166, 166, 166, 163, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138540616Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[1]R generation[0..12]: [92, 91, 93, 92, 92, 92, 90, 93, 93, 93, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138550925Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138561874Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[2]R flush_numbers[0..12]: [164, 163, 166, 164, 164, 164, 162, 166, 166, 166, 163, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138572472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[2]R generation[0..12]: [92, 91, 93, 92, 92, 92, 90, 93, 93, 93, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138582701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138593629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138603028Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138612697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138622146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138631485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"":"downstairs","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138641113Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138649862Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"0","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138660611Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"1","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.13867012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"client":"2","":"downstairs","session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138680098Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138689187Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"9d05fa42-991b-4fe8-8f76-cdf799ef3aeb is now active with session: 1e2ae5a7-51be-4c65-ab11-c2f2004d8126","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138699106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138707835Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"session_id":"1e2ae5a7-51be-4c65-ab11-c2f2004d8126"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138716884Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Activated sub_volume 9d05fa42-991b-4fe8-8f76-cdf799ef3aeb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138725142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.138733431Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.139077247Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"task":"crutest"} 01/30 Write block 267 len 8192 data: 7 7 02/30 Write block 463 len 12288 data: 8 11 10 03/30 Flush 04/30 Write block 536 len 8192 data: 8 9 05/30 Write block 489 len 40960 data: 5 8 9 7 8 8 8 7 6 6 06/30 Read block 50 len 8192 07/30 Write block 361 len 16384 data: 9 14 15 16 08/30 Write block 418 len 4096 data: 9 09/30 Read block 305 len 8192 10/30 Read block 424 len 36864 11/30 Read block 249 len 32768 12/30 Read block 166 len 8192 13/30 Read block 57 len 32768 14/30 Write block 577 len 8192 data: 8 7 15/30 Read block 372 len 24576 16/30 Read block 371 len 8192 17/30 Read block 143 len 36864 18/30 Read block 76 len 24576 19/30 Write block 109 len 40960 data: 8 8 6 6 8 8 9 6 7 6 20/30 Write block 164 len 32768 data: 7 6 6 6 7 6 5 9 21/30 Read block 13 len 12288 22/30 Read block 53 len 36864 23/30 Write block 493 len 36864 data: 9 9 9 8 7 7 6 6 6 24/30 Read block 399 len 28672 25/30 Write block 496 len 36864 data: 9 8 8 7 7 7 6 6 7 26/30 Write block 323 len 40960 data: 6 5 6 8 10 13 11 8 9 9 27/30 Write block 391 len 16384 data: 11 10 8 8 28/30 Write block 205 len 20480 data: 7 5 8 6 7 29/30 Write block 368 len 12288 data: 12 13 11 30/30 Write block 218 len 28672 data: 11 12 12 11 13 12 11 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 3 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 10 Sent Done Sent false 1026 Acked Write 4 Sent Done Sent false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 5 3 29 37 Done 25 27 1 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:28.147197637Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:28.147380703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1562,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 93 93 93 166 166 166 F F F 3 060-079 92 92 92 164 164 164 F F F 4 080-099 92 92 92 164 164 164 F F F 5 100-119 92 92 92 164 164 164 T T F <--- 6 120-139 90 90 90 162 162 162 F F F 7 140-159 93 93 93 166 166 166 F F F 8 160-179 93 93 93 166 166 166 T T F <--- 9 180-199 93 93 93 166 166 166 F F F 10 200-219 91 91 91 163 163 163 T T F <--- 11 220-239 88 88 88 158 158 158 T T F <--- 12 240-259 92 92 92 165 165 165 F F F 13 260-279 94 94 92 167 167 164 F F T <--- 14 280-299 93 93 93 166 166 166 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 87 87 87 156 156 156 T T F <--- 17 340-359 91 91 91 163 163 163 F F F 18 360-379 93 93 93 166 166 166 T T F <--- 19 380-399 87 87 87 156 156 156 T T F <--- 20 400-419 91 91 91 163 163 163 T T F <--- 21 420-439 91 91 91 163 163 163 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 94 91 167 167 163 F F F <--- 24 480-499 87 87 87 156 156 156 T T F <--- 25 500-519 88 88 88 159 159 159 T T F <--- 26 520-539 92 92 92 164 164 164 T T F <--- 27 540-559 92 92 92 164 164 164 F F F 28 560-579 91 91 91 163 163 163 T T F <--- 29 580-599 87 87 87 156 156 156 F F F Max gen: 94, Max flush: 167 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 93 93 93 166 166 166 F F F 3 060-079 92 92 92 164 164 164 F F F 4 080-099 92 92 92 164 164 164 F F F 5 100-119 95 95 95 168 168 168 F F F 6 120-139 90 90 90 162 162 162 F F F 7 140-159 93 93 93 166 166 166 F F F 8 160-179 95 95 95 168 168 168 F F F 9 180-199 93 93 93 166 166 166 F F F 10 200-219 95 95 95 168 168 168 F F F 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 94 94 94 167 167 167 F F F 14 280-299 93 93 93 166 166 166 F F F 15 300-319 87 87 87 156 156 156 F F F 16 320-339 95 95 95 168 168 168 F F F 17 340-359 91 91 91 163 163 163 F F F 18 360-379 95 95 95 168 168 168 F F F 19 380-399 95 95 95 168 168 168 F F F 20 400-419 95 95 95 168 168 168 F F F 21 420-439 91 91 91 163 163 163 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 94 94 167 167 167 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 95 95 95 168 168 168 F F F 26 520-539 95 95 95 168 168 168 F F F 27 540-559 92 92 92 164 164 164 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 95, Max flush: 168 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.417065349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.42517927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.425507967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.425542803Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.425552372Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.42556677Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Crucible 3cc455fe-7ac4-4d72-be7a-d15fab4b447c has session id: e287737c-a79c-4a5c-b0fc-de4ef8a21ad4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.425841634Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Upstairs opts: Upstairs UUID: 3cc455fe-7ac4-4d72-be7a-d15fab4b447c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.425864531Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Crucible stats registered with UUID: 3cc455fe-7ac4-4d72-be7a-d15fab4b447c","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.425877859Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:42.425887238Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426039838Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426070484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426081703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426213196Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426227714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"3cc455fe-7ac4-4d72-be7a-d15fab4b447c active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426267899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426277478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"0","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426287366Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"1","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426297275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"2","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426307124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426319132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426329321Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65139)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426379055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63391)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426390013Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56058)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.426501839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427044858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"1","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427072015Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"0","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427166593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"2","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427430909Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[0]R flush_numbers[0..12]: [164, 163, 166, 164, 164, 168, 162, 166, 168, 166, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427460635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[0]R generation[0..12]: [92, 91, 93, 92, 92, 95, 90, 93, 95, 93, 95, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427471973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427483282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[1]R flush_numbers[0..12]: [164, 163, 166, 164, 164, 168, 162, 166, 168, 166, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.42749425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[1]R generation[0..12]: [92, 91, 93, 92, 92, 95, 90, 93, 95, 93, 95, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427505089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427518777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[2]R flush_numbers[0..12]: [164, 163, 166, 164, 164, 168, 162, 166, 168, 166, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427529666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[2]R generation[0..12]: [92, 91, 93, 92, 92, 95, 90, 93, 95, 93, 95, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427540254Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427551513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427561282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.42757135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427580949Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427590538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"":"downstairs","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427600357Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427609315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"0","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427619124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"1","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427628873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"client":"2","":"downstairs","session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427638922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.4276483Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"3cc455fe-7ac4-4d72-be7a-d15fab4b447c is now active with session: e287737c-a79c-4a5c-b0fc-de4ef8a21ad4","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427658769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427667658Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"session_id":"e287737c-a79c-4a5c-b0fc-de4ef8a21ad4"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427678217Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Activated sub_volume 3cc455fe-7ac4-4d72-be7a-d15fab4b447c","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427692385Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427700784Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.427938843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"task":"crutest"} 01/30 Write block 304 len 24576 data: 7 7 10 10 10 13 02/30 Read block 292 len 28672 03/30 Read block 22 len 32768 04/30 Write block 61 len 36864 data: 10 10 9 8 7 8 7 7 6 05/30 Flush 06/30 Write block 219 len 4096 data: 13 07/30 Flush 08/30 Write block 516 len 40960 data: 9 13 10 11 13 12 12 10 8 8 09/30 Read block 358 len 12288 10/30 Read block 176 len 24576 11/30 Write block 78 len 28672 data: 4 5 5 4 5 4 6 12/30 Write block 89 len 12288 data: 5 6 5 13/30 Read block 133 len 12288 14/30 Read block 556 len 16384 15/30 Flush 16/30 Write block 115 len 8192 data: 10 7 17/30 Read block 22 len 20480 18/30 Read block 176 len 20480 19/30 Read block 411 len 28672 20/30 Read block 450 len 40960 21/30 Read block 572 len 24576 22/30 Write block 125 len 36864 data: 8 6 4 4 5 5 5 6 8 23/30 Write block 53 len 12288 data: 8 6 6 24/30 Read block 535 len 4096 25/30 Write block 436 len 4096 data: 8 26/30 Read block 555 len 36864 27/30 Write block 392 len 4096 data: 11 28/30 Write block 167 len 36864 data: 7 8 7 6 10 7 8 7 9 29/30 Read block 109 len 36864 30/30 Write block 144 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Sent false 1001 Acked Read 7 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:42.436550209Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:42.436746024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1571,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 93 93 93 166 166 166 T T F <--- 3 060-079 96 96 92 171 171 164 F F F <--- 4 080-099 96 96 92 171 171 164 F F F <--- 5 100-119 95 95 95 168 168 168 T T F <--- 6 120-139 90 90 90 162 162 162 T T F <--- 7 140-159 93 93 93 166 166 166 T T F <--- 8 160-179 95 95 95 168 168 168 T T F <--- 9 180-199 93 93 93 166 166 166 F F F 10 200-219 96 96 95 170 170 168 F F F <--- 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 94 94 94 167 167 167 F F F 14 280-299 93 93 93 166 166 166 F F F 15 300-319 96 96 87 169 169 156 F F F <--- 16 320-339 95 95 95 168 168 168 F F F 17 340-359 91 91 91 163 163 163 F F F 18 360-379 95 95 95 168 168 168 F F F 19 380-399 95 95 95 168 168 168 T T F <--- 20 400-419 95 95 95 168 168 168 F F F 21 420-439 91 91 91 163 163 163 T T F <--- 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 94 94 167 167 167 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 96 95 171 171 168 F F F <--- 26 520-539 96 96 95 171 171 168 F F F <--- 27 540-559 92 92 92 164 164 164 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 96, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 97 97 97 172 172 172 F F F 3 060-079 96 96 96 171 171 171 F F F 4 080-099 96 96 96 171 171 171 F F F 5 100-119 97 97 97 172 172 172 F F F 6 120-139 97 97 97 172 172 172 F F F 7 140-159 97 97 97 172 172 172 F F F 8 160-179 97 97 97 172 172 172 F F F 9 180-199 93 93 93 166 166 166 F F F 10 200-219 96 96 96 170 170 170 F F F 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 94 94 94 167 167 167 F F F 14 280-299 93 93 93 166 166 166 F F F 15 300-319 96 96 96 169 169 169 F F F 16 320-339 95 95 95 168 168 168 F F F 17 340-359 91 91 91 163 163 163 F F F 18 360-379 95 95 95 168 168 168 F F F 19 380-399 97 97 97 172 172 172 F F F 20 400-419 95 95 95 168 168 168 F F F 21 420-439 97 97 97 172 172 172 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 94 94 167 167 167 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 96 96 171 171 171 F F F 26 520-539 96 96 96 171 171 171 F F F 27 540-559 92 92 92 164 164 164 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 87 87 87 156 156 156 F F F Max gen: 97, Max flush: 172 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.696965044Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705066007Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705356579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705386285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705395174Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705406742Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Crucible 6c20738a-3627-418a-b2ce-9b9acc215f33 has session id: 091d2a67-db78-4882-91cc-1ffe9eeab6ed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705750948Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Upstairs opts: Upstairs UUID: 6c20738a-3627-418a-b2ce-9b9acc215f33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705788643Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Crucible stats registered with UUID: 6c20738a-3627-418a-b2ce-9b9acc215f33","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705799752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:56.70580976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705960891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.705991527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706002765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706144727Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706157195Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"6c20738a-3627-418a-b2ce-9b9acc215f33 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706191771Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.70620116Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"0","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706211198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"1","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706220837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"2","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706230476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706240545Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706272531Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:58514)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706289558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:39336)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706312615Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:54086)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706323054Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706949333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"0","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.706983099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"1","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707038451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"2","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707312536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[0]R flush_numbers[0..12]: [164, 163, 172, 171, 171, 172, 172, 172, 172, 166, 170, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707345972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[0]R generation[0..12]: [92, 91, 97, 96, 96, 97, 97, 97, 97, 93, 96, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.70735702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707368169Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[1]R flush_numbers[0..12]: [164, 163, 172, 171, 171, 172, 172, 172, 172, 166, 170, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707378987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[1]R generation[0..12]: [92, 91, 97, 96, 96, 97, 97, 97, 97, 93, 96, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707389336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707400115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[2]R flush_numbers[0..12]: [164, 163, 172, 171, 171, 172, 172, 172, 172, 166, 170, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707415863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[2]R generation[0..12]: [92, 91, 97, 96, 96, 97, 97, 97, 97, 93, 96, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707426181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.70743706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707446469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707456207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707466906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707476235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"":"downstairs","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707485794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707494532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"0","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707504001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"1","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.70751353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"client":"2","":"downstairs","session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707523099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707532178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"6c20738a-3627-418a-b2ce-9b9acc215f33 is now active with session: 091d2a67-db78-4882-91cc-1ffe9eeab6ed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707541986Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707550705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"session_id":"091d2a67-db78-4882-91cc-1ffe9eeab6ed"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707559754Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Activated sub_volume 6c20738a-3627-418a-b2ce-9b9acc215f33","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707567653Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707575612Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.707907009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"task":"crutest"} 01/30 Write block 315 len 24576 data: 9 7 8 6 3 4 02/30 Write block 269 len 16384 data: 6 7 8 7 03/30 Read block 468 len 16384 04/30 Read block 233 len 28672 05/30 Read block 516 len 12288 06/30 Read block 361 len 28672 07/30 Read block 560 len 8192 08/30 Read block 57 len 40960 09/30 Read block 429 len 36864 10/30 Write block 586 len 20480 data: 5 4 5 7 8 11/30 Read block 300 len 36864 12/30 Flush 13/30 Write block 265 len 4096 data: 4 14/30 Write block 584 len 28672 data: 5 6 6 5 6 8 9 15/30 Read block 517 len 24576 16/30 Read block 208 len 36864 17/30 Read block 584 len 28672 18/30 Read block 537 len 36864 19/30 Read block 120 len 24576 20/30 Read block 267 len 24576 21/30 Write block 47 len 40960 data: 11 11 11 9 11 11 9 7 7 4 22/30 Write block 337 len 16384 data: 7 5 6 8 23/30 Write block 417 len 36864 data: 9 10 7 5 5 5 5 4 4 24/30 Read block 522 len 8192 25/30 Write block 279 len 28672 data: 5 7 5 4 6 5 7 26/30 Write block 80 len 12288 data: 6 5 6 27/30 Write block 525 len 28672 data: 9 10 12 12 10 10 10 28/30 Write block 89 len 28672 data: 6 7 6 6 7 8 9 29/30 Write block 381 len 32768 data: 10 11 8 8 9 10 9 10 30/30 Write block 186 len 24576 data: 12 9 8 8 7 9 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Sent false 1001 Acked Write 4 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 30 36 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:57:56.715779561Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:57:56.715966057Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1580,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 97 97 97 172 172 172 T T F <--- 3 060-079 96 96 96 171 171 171 F F F 4 080-099 96 96 96 171 171 171 T T F <--- 5 100-119 97 97 97 172 172 172 F F F 6 120-139 97 97 97 172 172 172 F F F 7 140-159 97 97 97 172 172 172 F F F 8 160-179 97 97 97 172 172 172 F F F 9 180-199 93 93 93 166 166 166 T T F <--- 10 200-219 96 96 96 170 170 170 F F F 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 98 98 94 173 173 167 T T F <--- 14 280-299 93 93 93 166 166 166 T T F <--- 15 300-319 98 98 96 173 173 169 F F F <--- 16 320-339 98 98 95 173 173 168 T T F <--- 17 340-359 91 91 91 163 163 163 T T F <--- 18 360-379 95 95 95 168 168 168 F F F 19 380-399 97 97 97 172 172 172 T T F <--- 20 400-419 95 95 95 168 168 168 T T F <--- 21 420-439 97 97 97 172 172 172 T T F <--- 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 94 94 167 167 167 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 96 96 171 171 171 F F F 26 520-539 96 96 96 171 171 171 T T F <--- 27 540-559 92 92 92 164 164 164 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 98 98 87 173 173 156 T T F <--- Max gen: 98, Max flush: 173 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 99 99 99 174 174 174 F F F 3 060-079 96 96 96 171 171 171 F F F 4 080-099 99 99 99 174 174 174 F F F 5 100-119 97 97 97 172 172 172 F F F 6 120-139 97 97 97 172 172 172 F F F 7 140-159 97 97 97 172 172 172 F F F 8 160-179 97 97 97 172 172 172 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 96 96 96 170 170 170 F F F 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 99 99 99 174 174 174 F F F 14 280-299 99 99 99 174 174 174 F F F 15 300-319 98 98 98 173 173 173 F F F 16 320-339 99 99 99 174 174 174 F F F 17 340-359 99 99 99 174 174 174 F F F 18 360-379 95 95 95 168 168 168 F F F 19 380-399 99 99 99 174 174 174 F F F 20 400-419 99 99 99 174 174 174 F F F 21 420-439 99 99 99 174 174 174 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 94 94 167 167 167 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 96 96 171 171 171 F F F 26 520-539 99 99 99 174 174 174 F F F 27 540-559 92 92 92 164 164 164 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 99 99 99 174 174 174 F F F Max gen: 99, Max flush: 174 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.978915744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:10.986099864Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.986153477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.992904494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.9931675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993195777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993204505Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993216034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Crucible 83233199-0ec4-4a2d-adda-a94434f17beb has session id: 92063cc9-30ae-4fa1-989e-947739b8c0cc","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993615222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Upstairs opts: Upstairs UUID: 83233199-0ec4-4a2d-adda-a94434f17beb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.99363146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Crucible stats registered with UUID: 83233199-0ec4-4a2d-adda-a94434f17beb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993641599Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:10.993650598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993782991Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993799288Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993809197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993977455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.993991144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"83233199-0ec4-4a2d-adda-a94434f17beb active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994015121Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994024369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"0","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994033888Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"1","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994043507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"2","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994053026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994062974Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994073003Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52515)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994145954Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34259)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.9941709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54256)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994182579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994816157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"1","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994836394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"2","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.994853932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"0","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995216175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[0]R flush_numbers[0..12]: [164, 163, 174, 171, 174, 172, 172, 172, 172, 174, 170, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995242382Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[0]R generation[0..12]: [92, 91, 99, 96, 99, 97, 97, 97, 97, 99, 96, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.99525518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995266519Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[1]R flush_numbers[0..12]: [164, 163, 174, 171, 174, 172, 172, 172, 172, 174, 170, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995277207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[1]R generation[0..12]: [92, 91, 99, 96, 99, 97, 97, 97, 97, 99, 96, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995287636Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995298465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[2]R flush_numbers[0..12]: [164, 163, 174, 171, 174, 172, 172, 172, 172, 174, 170, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995308973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[2]R generation[0..12]: [92, 91, 99, 96, 99, 97, 97, 97, 97, 99, 96, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995319162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995329871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995339289Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995349038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995358307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995367536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"":"downstairs","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995377224Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995385943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"0","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995396742Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"1","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995406311Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"client":"2","":"downstairs","session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995415989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995424958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"83233199-0ec4-4a2d-adda-a94434f17beb is now active with session: 92063cc9-30ae-4fa1-989e-947739b8c0cc","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995434727Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995443496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"session_id":"92063cc9-30ae-4fa1-989e-947739b8c0cc"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995452595Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Activated sub_volume 83233199-0ec4-4a2d-adda-a94434f17beb","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995460534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995468663Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:10.995805489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} 01/30 Write block 62 len 8192 data: 11 10 02/30 Write block 274 len 28672 data: 9 10 10 8 6 6 8 03/30 Read block 514 len 4096 04/30 Flush 05/30 Write block 478 len 4096 data: 7 06/30 Read block 582 len 36864 07/30 Flush 08/30 Write block 551 len 12288 data: 7 7 8 09/30 Read block 546 len 32768 10/30 Read block 95 len 8192 11/30 Read block 472 len 8192 12/30 Read block 381 len 12288 13/30 Read block 31 len 16384 14/30 Read block 575 len 24576 15/30 Read block 264 len 32768 16/30 Read block 313 len 4096 17/30 Write block 393 len 4096 data: 9 18/30 Write block 301 len 40960 data: 8 7 6 8 8 11 11 11 14 11 19/30 Write block 580 len 32768 data: 7 4 4 5 6 7 7 6 20/30 Read block 240 len 12288 21/30 Flush 22/30 Write block 401 len 40960 data: 5 8 9 7 11 11 12 10 8 8 23/30 Flush 24/30 Write block 98 len 24576 data: 10 11 8 7 6 7 25/30 Read block 351 len 16384 26/30 Read block 426 len 16384 27/30 Write block 536 len 36864 data: 9 10 11 9 11 12 13 9 6 28/30 Write block 206 len 8192 data: 6 9 29/30 Write block 316 len 8192 data: 8 9 30/30 Read block 7 len 36864 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:11.004603621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:11.004803255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1589,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 99 99 99 174 174 174 F F F 3 060-079 96 100 100 171 175 175 T F F <--- 4 080-099 99 99 99 174 174 174 F T T <--- 5 100-119 97 97 97 172 172 172 F T T <--- 6 120-139 97 97 97 172 172 172 F F F 7 140-159 97 97 97 172 172 172 F F F 8 160-179 97 97 97 172 172 172 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 96 96 96 170 170 170 F T T <--- 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 99 100 100 174 175 175 F F F <--- 14 280-299 99 100 100 174 175 175 F F F <--- 15 300-319 98 100 100 173 177 177 F T T <--- 16 320-339 99 99 99 174 174 174 F F F 17 340-359 99 99 99 174 174 174 F F F 18 360-379 95 95 95 168 168 168 F F F 19 380-399 99 100 100 174 177 177 F F F <--- 20 400-419 99 100 100 174 178 178 F F F <--- 21 420-439 99 99 99 174 174 174 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 94 100 100 167 176 176 F F F <--- 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 96 96 171 171 171 F F F 26 520-539 99 99 99 174 174 174 F T T <--- 27 540-559 92 100 100 164 177 177 F T T <--- 28 560-579 95 95 95 168 168 168 F F F 29 580-599 99 100 100 174 177 177 F F F <--- Max gen: 100, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 91 91 163 163 163 F F F 2 040-059 99 99 99 174 174 174 F F F 3 060-079 100 100 100 175 175 175 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 101 101 101 179 179 179 F F F 6 120-139 97 97 97 172 172 172 F F F 7 140-159 97 97 97 172 172 172 F F F 8 160-179 97 97 97 172 172 172 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 95 95 95 168 168 168 F F F 12 240-259 92 92 92 165 165 165 F F F 13 260-279 100 100 100 175 175 175 F F F 14 280-299 100 100 100 175 175 175 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 99 99 99 174 174 174 F F F 17 340-359 99 99 99 174 174 174 F F F 18 360-379 95 95 95 168 168 168 F F F 19 380-399 100 100 100 177 177 177 F F F 20 400-419 100 100 100 178 178 178 F F F 21 420-439 99 99 99 174 174 174 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 96 96 171 171 171 F F F 26 520-539 101 101 101 179 179 179 F F F 27 540-559 101 101 101 179 179 179 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 100 100 100 177 177 177 F F F Max gen: 101, Max flush: 179 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.270266708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:25.277383147Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.27743595Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.284359815Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28462751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.284657186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.284667785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.284679463Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Crucible 8e881643-526c-4ee1-90be-61275b650ba8 has session id: 3ad1f10c-fee6-458e-8f73-5fe79c733585","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285062184Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Upstairs opts: Upstairs UUID: 8e881643-526c-4ee1-90be-61275b650ba8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28509349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Crucible stats registered with UUID: 8e881643-526c-4ee1-90be-61275b650ba8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285107168Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:25.285116217Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28524728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285263388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285273477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285443145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285456763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"8e881643-526c-4ee1-90be-61275b650ba8 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285491168Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285500557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"0","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285510336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"1","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285520045Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"2","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285531083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285541082Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285551011Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"ds_connection connected from Ok(127.0.0.1:39242)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285603134Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"ds_connection connected from Ok(127.0.0.1:52909)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285627931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"ds_connection connected from Ok(127.0.0.1:56323)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.285639229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286300344Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"1","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28633038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"2","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286417159Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"0","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286732198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[0]R flush_numbers[0..12]: [164, 163, 174, 175, 179, 179, 172, 172, 172, 174, 179, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286759504Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[0]R generation[0..12]: [92, 91, 99, 100, 101, 101, 97, 97, 97, 99, 101, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286770853Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286781951Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[1]R flush_numbers[0..12]: [164, 163, 174, 175, 179, 179, 172, 172, 172, 174, 179, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28679272Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[1]R generation[0..12]: [92, 91, 99, 100, 101, 101, 97, 97, 97, 99, 101, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286803299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286816387Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[2]R flush_numbers[0..12]: [164, 163, 174, 175, 179, 179, 172, 172, 172, 174, 179, 168]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286827256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[2]R generation[0..12]: [92, 91, 99, 100, 101, 101, 97, 97, 97, 99, 101, 95]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286837724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286848823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286858192Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28686829Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286877739Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286887158Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"":"downstairs","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286897317Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286906275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"0","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286916314Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"1","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286926233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"client":"2","":"downstairs","session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286936301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.28694556Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"8e881643-526c-4ee1-90be-61275b650ba8 is now active with session: 3ad1f10c-fee6-458e-8f73-5fe79c733585","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286955679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286965828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"session_id":"3ad1f10c-fee6-458e-8f73-5fe79c733585"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286980836Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Activated sub_volume 8e881643-526c-4ee1-90be-61275b650ba8","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286988945Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.286997204Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.287245392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} 01/30 Write block 590 len 16384 data: 10 8 10 11 02/30 Read block 308 len 28672 03/30 Write block 506 len 32768 data: 8 8 9 9 10 14 13 12 04/30 Write block 152 len 28672 data: 4 4 7 6 7 9 11 05/30 Read block 20 len 20480 06/30 Read block 555 len 12288 07/30 Write block 24 len 28672 data: 7 7 8 7 6 7 4 08/30 Write block 71 len 4096 data: 5 09/30 Read block 130 len 40960 10/30 Read block 40 len 32768 11/30 Flush 12/30 Write block 67 len 40960 data: 8 8 7 6 6 5 4 4 5 4 13/30 Read block 592 len 4096 14/30 Read block 414 len 40960 15/30 Write block 536 len 36864 data: 10 11 12 10 12 13 14 10 7 16/30 Read block 476 len 32768 17/30 Write block 364 len 20480 data: 17 13 14 13 13 18/30 Read block 518 len 4096 19/30 Read block 377 len 12288 20/30 Write block 240 len 12288 data: 5 7 9 21/30 Read block 149 len 28672 22/30 Flush 23/30 Write block 232 len 32768 data: 4 4 2 7 6 7 8 7 24/30 Read block 172 len 28672 25/30 Write block 124 len 8192 data: 9 9 26/30 Write block 47 len 20480 data: 12 12 12 10 12 27/30 Read block 215 len 8192 28/30 Write block 409 len 4096 data: 9 29/30 Read block 202 len 12288 30/30 Write block 556 len 8192 data: 8 6 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:25.297191815Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:25.297380001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1598,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 91 102 102 163 180 180 F F F <--- 2 040-059 99 99 99 174 174 174 F T T <--- 3 060-079 100 102 102 175 181 181 F F F <--- 4 080-099 101 101 101 179 179 179 F F F 5 100-119 101 101 101 179 179 179 F F F 6 120-139 97 97 97 172 172 172 F T T <--- 7 140-159 97 102 102 172 180 180 F F F <--- 8 160-179 97 97 97 172 172 172 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 95 95 95 168 168 168 F T T <--- 12 240-259 92 102 102 165 181 181 F F F <--- 13 260-279 100 100 100 175 175 175 F F F 14 280-299 100 100 100 175 175 175 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 99 99 99 174 174 174 F F F 17 340-359 99 99 99 174 174 174 F F F 18 360-379 95 102 102 168 181 181 F F F <--- 19 380-399 100 100 100 177 177 177 F F F 20 400-419 100 100 100 178 178 178 F T T <--- 21 420-439 99 99 99 174 174 174 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 96 102 102 171 180 180 F F F <--- 26 520-539 101 102 102 179 181 181 F F F <--- 27 540-559 101 102 102 179 181 181 F T T <--- 28 560-579 95 95 95 168 168 168 F F F 29 580-599 100 102 102 177 180 180 F F F <--- Max gen: 102, Max flush: 181 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 102 102 102 180 180 180 F F F 2 040-059 103 103 103 182 182 182 F F F 3 060-079 102 102 102 181 181 181 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 101 101 101 179 179 179 F F F 6 120-139 103 103 103 182 182 182 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 97 97 97 172 172 172 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 103 103 103 182 182 182 F F F 12 240-259 102 102 102 181 181 181 F F F 13 260-279 100 100 100 175 175 175 F F F 14 280-299 100 100 100 175 175 175 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 99 99 99 174 174 174 F F F 17 340-359 99 99 99 174 174 174 F F F 18 360-379 102 102 102 181 181 181 F F F 19 380-399 100 100 100 177 177 177 F F F 20 400-419 103 103 103 182 182 182 F F F 21 420-439 99 99 99 174 174 174 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 100 100 100 176 176 176 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 102 102 102 180 180 180 F F F 26 520-539 102 102 102 181 181 181 F F F 27 540-559 103 103 103 182 182 182 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 102 102 102 180 180 180 F F F Max gen: 103, Max flush: 182 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.558016537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.566098312Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.566394574Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.56642248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.566431359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.566443287Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Crucible be0af543-cec9-4e32-b551-0c0ebb2e8815 has session id: 6df763da-3389-44ec-ad6a-92b6cfd35ba1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.566860283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Upstairs opts: Upstairs UUID: be0af543-cec9-4e32-b551-0c0ebb2e8815, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.566876871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Crucible stats registered with UUID: be0af543-cec9-4e32-b551-0c0ebb2e8815","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.56688684Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:39.566898089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567033471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567050479Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567060537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.56726944Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567283509Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"be0af543-cec9-4e32-b551-0c0ebb2e8815 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567317614Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567326873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"0","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567336492Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"1","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567346001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"2","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567355489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567365308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567375057Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50724)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567416331Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48498)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567481873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54373)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.567493771Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568205079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"0","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568228256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"2","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568239705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"1","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568559954Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[0]R flush_numbers[0..12]: [164, 180, 182, 181, 179, 179, 182, 180, 172, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568579421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[0]R generation[0..12]: [92, 102, 103, 102, 101, 101, 103, 102, 97, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.56859028Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568601608Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[1]R flush_numbers[0..12]: [164, 180, 182, 181, 179, 179, 182, 180, 172, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568612317Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[1]R generation[0..12]: [92, 102, 103, 102, 101, 101, 103, 102, 97, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568622715Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568633494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[2]R flush_numbers[0..12]: [164, 180, 182, 181, 179, 179, 182, 180, 172, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568644033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[2]R generation[0..12]: [92, 102, 103, 102, 101, 101, 103, 102, 97, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568654501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.56866538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568674709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568686497Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568695876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568705155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"":"downstairs","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568714824Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568723532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"0","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568733091Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"1","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.56874256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"client":"2","":"downstairs","session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568752309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568761557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"be0af543-cec9-4e32-b551-0c0ebb2e8815 is now active with session: 6df763da-3389-44ec-ad6a-92b6cfd35ba1","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568771616Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568780345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"session_id":"6df763da-3389-44ec-ad6a-92b6cfd35ba1"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568789534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Activated sub_volume be0af543-cec9-4e32-b551-0c0ebb2e8815","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568797563Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.568805602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.569146698Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"task":"crutest"} 01/30 Write block 320 len 8192 data: 5 5 02/30 Read block 22 len 28672 03/30 Write block 507 len 12288 data: 9 10 10 04/30 Read block 194 len 12288 05/30 Flush 06/30 Flush 07/30 Write block 383 len 40960 data: 9 9 10 11 10 11 13 12 12 12 08/30 Read block 459 len 32768 09/30 Read block 319 len 28672 10/30 Read block 71 len 4096 11/30 Write block 462 len 36864 data: 8 9 12 11 8 10 11 11 11 12/30 Read block 592 len 24576 13/30 Read block 417 len 8192 14/30 Read block 451 len 40960 15/30 Read block 347 len 8192 16/30 Write block 141 len 40960 data: 4 5 4 5 5 5 3 4 6 5 17/30 Read block 221 len 8192 18/30 Write block 351 len 20480 data: 7 5 6 7 6 19/30 Write block 179 len 4096 data: 6 20/30 Read block 569 len 40960 21/30 Write block 146 len 12288 data: 6 4 5 22/30 Flush 23/30 Write block 432 len 32768 data: 8 8 10 8 9 10 10 10 24/30 Read block 85 len 8192 25/30 Write block 528 len 4096 data: 13 26/30 Read block 85 len 32768 27/30 Write block 67 len 4096 data: 9 28/30 Read block 444 len 12288 29/30 Read block 332 len 8192 30/30 Write block 426 len 16384 data: 5 6 5 6 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:39.578000163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:39.578236142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1607,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 102 102 102 180 180 180 F F F 2 040-059 103 103 103 182 182 182 F F F 3 060-079 102 102 102 181 181 181 T F T <--- 4 080-099 101 101 101 179 179 179 F F F 5 100-119 101 101 101 179 179 179 F F F 6 120-139 103 103 103 182 182 182 F F F 7 140-159 104 102 104 185 180 185 F F F <--- 8 160-179 104 97 104 185 172 185 F F F <--- 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 103 103 103 182 182 182 F F F 12 240-259 102 102 102 181 181 181 F F F 13 260-279 100 100 100 175 175 175 F F F 14 280-299 100 100 100 175 175 175 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 104 99 104 183 174 183 F T F <--- 17 340-359 104 99 104 185 174 185 F F F <--- 18 360-379 102 102 102 181 181 181 F F F 19 380-399 104 100 104 185 177 185 F F F <--- 20 400-419 103 103 103 182 182 182 F F F 21 420-439 99 99 99 174 174 174 T F T <--- 22 440-459 92 92 92 164 164 164 F F F 23 460-479 104 100 104 185 176 185 F F F <--- 24 480-499 95 95 95 168 168 168 F F F 25 500-519 104 102 104 183 180 183 F F F <--- 26 520-539 102 102 102 181 181 181 T F T <--- 27 540-559 103 103 103 182 182 182 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 102 102 102 180 180 180 F F F Max gen: 104, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 102 102 102 180 180 180 F F F 2 040-059 103 103 103 182 182 182 F F F 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 101 101 101 179 179 179 F F F 6 120-139 103 103 103 182 182 182 F F F 7 140-159 104 104 104 185 185 185 F F F 8 160-179 104 104 104 185 185 185 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 103 103 103 182 182 182 F F F 12 240-259 102 102 102 181 181 181 F F F 13 260-279 100 100 100 175 175 175 F F F 14 280-299 100 100 100 175 175 175 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 104 104 104 183 183 183 F F F 17 340-359 104 104 104 185 185 185 F F F 18 360-379 102 102 102 181 181 181 F F F 19 380-399 104 104 104 185 185 185 F F F 20 400-419 103 103 103 182 182 182 F F F 21 420-439 105 105 105 186 186 186 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 104 104 104 185 185 185 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 104 104 104 183 183 183 F F F 26 520-539 105 105 105 186 186 186 F F F 27 540-559 103 103 103 182 182 182 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 102 102 102 180 180 180 F F F Max gen: 105, Max flush: 186 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.838008Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846209819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846505841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846554905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846563903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846575442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Crucible 504c98d2-00c1-4a7f-8dd1-6239e830b429 has session id: 8a42fa56-cec8-4c73-ba1d-351e25e118da","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.84689891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Upstairs opts: Upstairs UUID: 504c98d2-00c1-4a7f-8dd1-6239e830b429, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846958572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Crucible stats registered with UUID: 504c98d2-00c1-4a7f-8dd1-6239e830b429","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.846968611Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:53.84697771Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847107823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847139989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847150967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847293949Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847310797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"504c98d2-00c1-4a7f-8dd1-6239e830b429 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847341943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847353351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"0","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.84736337Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"1","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847373279Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"2","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847383037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847393116Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847403015Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"ds_connection connected from Ok(127.0.0.1:58921)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847419833Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"ds_connection connected from Ok(127.0.0.1:65504)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847470226Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"ds_connection connected from Ok(127.0.0.1:35961)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.847572473Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.84813268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"0","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848184744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"1","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848225228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"2","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848539788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[0]R flush_numbers[0..12]: [164, 180, 182, 186, 179, 179, 182, 185, 185, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848567124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[0]R generation[0..12]: [92, 102, 103, 105, 101, 101, 103, 104, 104, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848578843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848592901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[1]R flush_numbers[0..12]: [164, 180, 182, 186, 179, 179, 182, 185, 185, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.84860363Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[1]R generation[0..12]: [92, 102, 103, 105, 101, 101, 103, 104, 104, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848614328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848625377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[2]R flush_numbers[0..12]: [164, 180, 182, 186, 179, 179, 182, 185, 185, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848635935Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[2]R generation[0..12]: [92, 102, 103, 105, 101, 101, 103, 104, 104, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848646414Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848657283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848666741Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.84867649Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848686179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848695738Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"":"downstairs","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848705496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848714315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"0","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848724174Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"1","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848735083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"client":"2","":"downstairs","session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848745041Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.84875435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"504c98d2-00c1-4a7f-8dd1-6239e830b429 is now active with session: 8a42fa56-cec8-4c73-ba1d-351e25e118da","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848764499Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848773308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"session_id":"8a42fa56-cec8-4c73-ba1d-351e25e118da"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848782426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Activated sub_volume 504c98d2-00c1-4a7f-8dd1-6239e830b429","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848790615Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.848798734Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.849128672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"task":"crutest"} 01/30 Write block 420 len 16384 data: 6 6 6 6 02/30 Read block 155 len 28672 03/30 Write block 345 len 36864 data: 5 6 6 5 7 9 8 6 7 04/30 Read block 428 len 4096 05/30 Read block 586 len 36864 06/30 Write block 278 len 20480 data: 7 7 9 6 5 07/30 Write block 114 len 4096 data: 9 08/30 Read block 55 len 28672 09/30 Read block 533 len 20480 10/30 Write block 154 len 32768 data: 8 7 8 10 12 11 10 9 11/30 Read block 161 len 40960 12/30 Read block 325 len 40960 13/30 Write block 271 len 32768 data: 9 8 9 10 11 11 9 8 14/30 Read block 249 len 28672 15/30 Read block 376 len 40960 16/30 Write block 244 len 32768 data: 10 8 9 8 7 7 8 8 17/30 Write block 281 len 12288 data: 7 6 7 18/30 Read block 549 len 12288 19/30 Write block 147 len 24576 data: 5 6 7 6 5 5 20/30 Read block 10 len 28672 21/30 Write block 248 len 24576 data: 8 8 9 9 7 9 22/30 Read block 465 len 24576 23/30 Write block 325 len 28672 data: 7 9 11 14 12 9 10 24/30 Write block 502 len 24576 data: 7 7 8 6 9 10 25/30 Write block 419 len 40960 data: 8 7 7 7 7 5 5 6 7 6 26/30 Read block 578 len 20480 27/30 Read block 20 len 16384 28/30 Write block 592 len 8192 data: 11 12 29/30 Write block 467 len 28672 data: 11 12 12 12 13 14 12 30/30 Write block 366 len 36864 data: 15 14 14 14 12 8 8 6 5 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 25 29 Done 28 28 5 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:58:53.858546583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:58:53.858736369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1616,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 102 102 102 180 180 180 F F F 2 040-059 103 103 103 182 182 182 F F F 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 101 101 101 179 179 179 T T F <--- 6 120-139 103 103 103 182 182 182 F F F 7 140-159 104 104 104 185 185 185 T T F <--- 8 160-179 104 104 104 185 185 185 T T F <--- 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 103 103 103 182 182 182 F F F 12 240-259 102 102 102 181 181 181 T T F <--- 13 260-279 100 100 100 175 175 175 T T F <--- 14 280-299 100 100 100 175 175 175 T T F <--- 15 300-319 101 101 101 179 179 179 F F F 16 320-339 104 104 104 183 183 183 T T F <--- 17 340-359 104 104 104 185 185 185 T T T 18 360-379 102 102 102 181 181 181 T T F <--- 19 380-399 104 104 104 185 185 185 F F F 20 400-419 103 103 103 182 182 182 T T F <--- 21 420-439 105 105 105 186 186 186 T T T 22 440-459 92 92 92 164 164 164 F F F 23 460-479 104 104 104 185 185 185 T T F <--- 24 480-499 95 95 95 168 168 168 F F F 25 500-519 104 104 104 183 183 183 T T F <--- 26 520-539 105 105 105 186 186 186 F F F 27 540-559 103 103 103 182 182 182 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 102 102 102 180 180 180 T T F <--- Max gen: 105, Max flush: 186 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 164 164 164 F F F 1 020-039 102 102 102 180 180 180 F F F 2 040-059 103 103 103 182 182 182 F F F 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 106 106 106 187 187 187 F F F 6 120-139 103 103 103 182 182 182 F F F 7 140-159 106 106 106 187 187 187 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F F F 11 220-239 103 103 103 182 182 182 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 187 187 187 F F F 14 280-299 106 106 106 187 187 187 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 106 106 106 187 187 187 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 104 104 104 185 185 185 F F F 20 400-419 106 106 106 187 187 187 F F F 21 420-439 106 106 106 187 187 187 F F F 22 440-459 92 92 92 164 164 164 F F F 23 460-479 106 106 106 187 187 187 F F F 24 480-499 95 95 95 168 168 168 F F F 25 500-519 106 106 106 187 187 187 F F F 26 520-539 105 105 105 186 186 186 F F F 27 540-559 103 103 103 182 182 182 F F F 28 560-579 95 95 95 168 168 168 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 106, Max flush: 187 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.125716845Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:08.132848823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.132902866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.139776157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140038903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140073308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140084547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140096345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Crucible 98c2f471-e321-4b08-b0e7-ff61a08eb7ae has session id: 79e52fac-7abd-494a-a31d-06077f2cdaae","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140418573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Upstairs opts: Upstairs UUID: 98c2f471-e321-4b08-b0e7-ff61a08eb7ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140439941Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Crucible stats registered with UUID: 98c2f471-e321-4b08-b0e7-ff61a08eb7ae","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140450379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:08.140459818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140579033Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.14059673Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140607089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140778477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140791065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"98c2f471-e321-4b08-b0e7-ff61a08eb7ae active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140819262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.14082863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"0","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140838669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"1","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140848448Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"2","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140858277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140869875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140880024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"ds_connection connected from Ok(127.0.0.1:44201)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140948975Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"ds_connection connected from Ok(127.0.0.1:50112)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.140976641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"ds_connection connected from Ok(127.0.0.1:65215)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.14098804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.141660583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"1","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.141726274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"0","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.141738173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"2","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142082698Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[0]R flush_numbers[0..12]: [164, 180, 182, 186, 179, 187, 182, 187, 187, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142117404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[0]R generation[0..12]: [92, 102, 103, 105, 101, 106, 103, 106, 106, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142128702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142139961Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[1]R flush_numbers[0..12]: [164, 180, 182, 186, 179, 187, 182, 187, 187, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142150829Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[1]R generation[0..12]: [92, 102, 103, 105, 101, 106, 103, 106, 106, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142161318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142174676Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[2]R flush_numbers[0..12]: [164, 180, 182, 186, 179, 187, 182, 187, 187, 174, 179, 182]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142185295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[2]R generation[0..12]: [92, 102, 103, 105, 101, 106, 103, 106, 106, 99, 101, 103]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142195844Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142206672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Max found gen is 107","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142216091Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Generation requested: 108 >= found:107","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.14222587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142235468Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142244897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"":"downstairs","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142254706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142263555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"0","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142273234Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"1","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142282862Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"client":"2","":"downstairs","session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142292701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.14230186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"98c2f471-e321-4b08-b0e7-ff61a08eb7ae is now active with session: 79e52fac-7abd-494a-a31d-06077f2cdaae","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142311909Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142321877Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"session_id":"79e52fac-7abd-494a-a31d-06077f2cdaae"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142331056Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Activated sub_volume 98c2f471-e321-4b08-b0e7-ff61a08eb7ae","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142339335Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142347514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.142645865Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} 01/30 Write block 470 len 28672 data: 13 14 15 13 8 9 8 02/30 Read block 451 len 40960 03/30 Read block 206 len 36864 04/30 Flush 05/30 Write block 480 len 4096 data: 7 06/30 Write block 457 len 32768 data: 6 8 9 8 9 9 10 13 07/30 Flush 08/30 Write block 44 len 4096 data: 11 09/30 Flush 10/30 Write block 100 len 40960 data: 9 8 7 8 8 9 10 10 10 9 11/30 Write block 251 len 40960 data: 10 8 10 6 5 7 6 5 6 7 12/30 Read block 358 len 4096 13/30 Read block 52 len 20480 14/30 Flush 15/30 Write block 19 len 32768 data: 5 4 5 5 7 8 8 9 16/30 Read block 248 len 12288 17/30 Read block 348 len 28672 18/30 Read block 468 len 40960 19/30 Flush 20/30 Write block 558 len 28672 data: 8 10 12 12 13 9 12 21/30 Read block 375 len 40960 22/30 Read block 268 len 12288 23/30 Read block 203 len 8192 24/30 Read block 143 len 12288 25/30 Flush 26/30 Write block 218 len 40960 data: 12 14 13 12 14 13 12 13 11 11 27/30 Write block 285 len 20480 data: 8 10 10 13 11 28/30 Read block 117 len 36864 29/30 Read block 157 len 4096 30/30 Write block 352 len 20480 data: 7 8 8 7 6 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:08.152705624Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:08.152885961Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1625,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 108 108 164 192 192 F F F <--- 1 020-039 102 108 108 180 192 192 F F F <--- 2 040-059 103 108 108 182 190 190 F F F <--- 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 106 108 108 187 191 191 F F F <--- 6 120-139 103 103 103 182 182 182 F F F 7 140-159 106 106 106 187 187 187 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 101 101 101 179 179 179 F T T <--- 11 220-239 103 103 103 182 182 182 F T T <--- 12 240-259 106 108 108 187 191 191 F F F <--- 13 260-279 106 108 108 187 191 191 F F F <--- 14 280-299 106 106 106 187 187 187 F T T <--- 15 300-319 101 101 101 179 179 179 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 106 106 106 187 187 187 F T T <--- 18 360-379 106 106 106 187 187 187 F F F 19 380-399 104 104 104 185 185 185 F F F 20 400-419 106 106 106 187 187 187 F F F 21 420-439 106 106 106 187 187 187 F F F 22 440-459 92 108 108 164 189 189 F F F <--- 23 460-479 106 108 108 187 189 189 F F F <--- 24 480-499 95 108 108 168 189 189 F F F <--- 25 500-519 106 106 106 187 187 187 F F F 26 520-539 105 105 105 186 186 186 F F F 27 540-559 103 108 108 182 193 193 F F F <--- 28 560-579 95 108 108 168 193 193 F F F <--- 29 580-599 106 106 106 187 187 187 F F F Max gen: 108, Max flush: 193 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 192 192 192 F F F 1 020-039 108 108 108 192 192 192 F F F 2 040-059 108 108 108 190 190 190 F F F 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 108 108 108 191 191 191 F F F 6 120-139 103 103 103 182 182 182 F F F 7 140-159 106 106 106 187 187 187 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 99 99 99 174 174 174 F F F 10 200-219 109 109 109 194 194 194 F F F 11 220-239 109 109 109 194 194 194 F F F 12 240-259 108 108 108 191 191 191 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 109 109 109 194 194 194 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 106 106 106 187 187 187 F F F 17 340-359 109 109 109 194 194 194 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 104 104 104 185 185 185 F F F 20 400-419 106 106 106 187 187 187 F F F 21 420-439 106 106 106 187 187 187 F F F 22 440-459 108 108 108 189 189 189 F F F 23 460-479 108 108 108 189 189 189 F F F 24 480-499 108 108 108 189 189 189 F F F 25 500-519 106 106 106 187 187 187 F F F 26 520-539 105 105 105 186 186 186 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 109, Max flush: 194 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.417197142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.425367605Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.425670006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.425700672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.425709491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.42572115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Crucible 85ecf7dc-cac8-4e15-ab16-037284c0f579 has session id: 5c61cc93-3c17-4522-b965-ff0d1fd2ce7a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426130467Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Upstairs opts: Upstairs UUID: 85ecf7dc-cac8-4e15-ab16-037284c0f579, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426157813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Crucible stats registered with UUID: 85ecf7dc-cac8-4e15-ab16-037284c0f579","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426168822Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:22.426178171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426302464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426332831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426343829Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.42648883Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426506118Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"85ecf7dc-cac8-4e15-ab16-037284c0f579 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426533445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426543053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"0","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426552752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"1","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426562481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"2","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.42657209Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426582038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426592017Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48987)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426628622Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44349)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426690804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47625)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.426771024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427354728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"0","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427368197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"2","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427378885Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"1","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427717112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[0]R flush_numbers[0..12]: [192, 192, 190, 186, 179, 191, 182, 187, 187, 174, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427742718Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[0]R generation[0..12]: [108, 108, 108, 105, 101, 108, 103, 106, 106, 99, 109, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427753867Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427764655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[1]R flush_numbers[0..12]: [192, 192, 190, 186, 179, 191, 182, 187, 187, 174, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427775064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[1]R generation[0..12]: [108, 108, 108, 105, 101, 108, 103, 106, 106, 99, 109, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427785563Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427796361Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[2]R flush_numbers[0..12]: [192, 192, 190, 186, 179, 191, 182, 187, 187, 174, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.42780699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[2]R generation[0..12]: [108, 108, 108, 105, 101, 108, 103, 106, 106, 99, 109, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427817679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427828537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427837806Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427849034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427858343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427867632Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"":"downstairs","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427877341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.42788602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"0","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427895818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"1","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427905377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"client":"2","":"downstairs","session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427915186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427924235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"85ecf7dc-cac8-4e15-ab16-037284c0f579 is now active with session: 5c61cc93-3c17-4522-b965-ff0d1fd2ce7a","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427934243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427943142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"session_id":"5c61cc93-3c17-4522-b965-ff0d1fd2ce7a"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427952381Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Activated sub_volume 85ecf7dc-cac8-4e15-ab16-037284c0f579","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.42796038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.427968409Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.428323273Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"task":"crutest"} 01/30 Write block 281 len 28672 data: 8 7 8 6 9 11 11 02/30 Read block 457 len 16384 03/30 Read block 426 len 28672 04/30 Flush 05/30 Write block 511 len 8192 data: 15 14 06/30 Write block 517 len 40960 data: 14 11 12 14 13 13 11 9 10 11 07/30 Read block 222 len 20480 08/30 Write block 30 len 28672 data: 5 5 7 7 7 8 9 09/30 Write block 475 len 24576 data: 10 9 7 8 8 8 10/30 Write block 216 len 8192 data: 6 8 11/30 Write block 417 len 24576 data: 10 11 9 8 8 8 12/30 Write block 331 len 16384 data: 11 10 9 9 13/30 Write block 348 len 32768 data: 6 8 10 9 8 9 9 8 14/30 Read block 580 len 16384 15/30 Flush 16/30 Write block 262 len 12288 data: 6 7 5 17/30 Write block 405 len 24576 data: 12 12 13 11 10 9 18/30 Read block 77 len 32768 19/30 Read block 62 len 4096 20/30 Read block 59 len 16384 21/30 Read block 352 len 20480 22/30 Flush 23/30 Write block 490 len 24576 data: 9 10 8 10 10 10 24/30 Write block 36 len 20480 data: 10 7 7 7 9 25/30 Write block 16 len 32768 data: 7 7 6 6 5 6 6 8 26/30 Read block 189 len 8192 27/30 Write block 283 len 32768 data: 9 7 10 12 12 14 12 10 28/30 Write block 124 len 32768 data: 10 10 7 5 5 6 6 6 29/30 Write block 177 len 16384 data: 10 9 7 8 30/30 Read block 312 len 28672 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 26 0 25 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1003)) Some(JobId(1021)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:22.436824953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:22.437015999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1634,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 192 192 192 T F T <--- 1 020-039 110 108 110 196 192 196 T F T <--- 2 040-059 108 108 108 190 190 190 T F T <--- 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 108 108 108 191 191 191 F F F 6 120-139 103 103 103 182 182 182 T F T <--- 7 140-159 106 106 106 187 187 187 F F F 8 160-179 106 106 106 187 187 187 T F T <--- 9 180-199 99 99 99 174 174 174 T F T <--- 10 200-219 110 109 110 196 194 196 F F F <--- 11 220-239 109 109 109 194 194 194 F F F 12 240-259 108 108 108 191 191 191 F F F 13 260-279 110 108 110 197 191 197 F F F <--- 14 280-299 110 110 110 195 195 195 T F T <--- 15 300-319 101 101 101 179 179 179 F F F 16 320-339 110 106 110 196 187 196 F F F <--- 17 340-359 110 109 110 196 194 196 F F F <--- 18 360-379 106 106 106 187 187 187 F F F 19 380-399 104 104 104 185 185 185 F F F 20 400-419 110 106 110 197 187 197 F F F <--- 21 420-439 110 106 110 196 187 196 F F F <--- 22 440-459 108 108 108 189 189 189 F F F 23 460-479 110 108 110 196 189 196 F F F <--- 24 480-499 110 108 110 196 189 196 T F T <--- 25 500-519 110 106 110 196 187 196 F F F <--- 26 520-539 110 105 110 196 186 196 F F F <--- 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 110, Max flush: 197 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 198 198 198 F F F 1 020-039 111 111 111 198 198 198 F F F 2 040-059 111 111 111 198 198 198 F F F 3 060-079 105 105 105 186 186 186 F F F 4 080-099 101 101 101 179 179 179 F F F 5 100-119 108 108 108 191 191 191 F F F 6 120-139 111 111 111 198 198 198 F F F 7 140-159 106 106 106 187 187 187 F F F 8 160-179 111 111 111 198 198 198 F F F 9 180-199 111 111 111 198 198 198 F F F 10 200-219 110 110 110 196 196 196 F F F 11 220-239 109 109 109 194 194 194 F F F 12 240-259 108 108 108 191 191 191 F F F 13 260-279 110 110 110 197 197 197 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 110 110 110 196 196 196 F F F 17 340-359 110 110 110 196 196 196 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 104 104 104 185 185 185 F F F 20 400-419 110 110 110 197 197 197 F F F 21 420-439 110 110 110 196 196 196 F F F 22 440-459 108 108 108 189 189 189 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 110 110 110 196 196 196 F F F 26 520-539 110 110 110 196 196 196 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 111, Max flush: 198 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.707409943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716012261Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716315452Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716347547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716356516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716368575Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Crucible 31403b74-4d6e-4cdf-96fc-7ab740aee769 has session id: 66762293-d2e9-4752-8eef-1260e19e788d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716772852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Upstairs opts: Upstairs UUID: 31403b74-4d6e-4cdf-96fc-7ab740aee769, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71679169Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Crucible stats registered with UUID: 31403b74-4d6e-4cdf-96fc-7ab740aee769","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716802569Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:36.716812557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71694694Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716964418Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.716974956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717157373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717170861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"31403b74-4d6e-4cdf-96fc-7ab740aee769 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717212565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717222394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"0","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717234283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"1","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717244381Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"2","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71725464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717265299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717275737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64839)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717349048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46350)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717377194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41502)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.717389233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71802492Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"2","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718071794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"1","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718085233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"0","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718449136Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[0]R flush_numbers[0..12]: [198, 198, 198, 186, 179, 191, 198, 187, 198, 198, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718476192Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[0]R generation[0..12]: [111, 111, 111, 105, 101, 108, 111, 106, 111, 111, 110, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71848841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718502409Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[1]R flush_numbers[0..12]: [198, 198, 198, 186, 179, 191, 198, 187, 198, 198, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718513737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[1]R generation[0..12]: [111, 111, 111, 105, 101, 108, 111, 106, 111, 111, 110, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718524896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718536384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[2]R flush_numbers[0..12]: [198, 198, 198, 186, 179, 191, 198, 187, 198, 198, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718547693Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[2]R generation[0..12]: [111, 111, 111, 105, 101, 108, 111, 106, 111, 111, 110, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718558881Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71857043Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718580369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718590807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718600856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718610995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"":"downstairs","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718621633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718631072Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"0","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718641581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"1","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718652099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"client":"2","":"downstairs","session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718663818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718673597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"31403b74-4d6e-4cdf-96fc-7ab740aee769 is now active with session: 66762293-d2e9-4752-8eef-1260e19e788d","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718684145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718693434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"session_id":"66762293-d2e9-4752-8eef-1260e19e788d"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718703153Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Activated sub_volume 31403b74-4d6e-4cdf-96fc-7ab740aee769","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.718711752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.71872049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.719037079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"task":"crutest"} 01/30 Write block 266 len 40960 data: 6 8 8 7 8 10 9 10 11 12 02/30 Read block 473 len 28672 03/30 Write block 152 len 12288 data: 6 5 9 04/30 Read block 328 len 4096 05/30 Flush 06/30 Write block 448 len 20480 data: 9 9 9 7 7 07/30 Read block 502 len 36864 08/30 Write block 82 len 24576 data: 7 5 7 7 7 7 09/30 Read block 325 len 36864 10/30 Flush 11/30 Write block 74 len 36864 data: 5 6 5 4 5 6 7 6 8 12/30 Write block 87 len 8192 data: 8 8 13/30 Write block 243 len 8192 data: 9 11 14/30 Write block 99 len 8192 data: 12 10 15/30 Read block 173 len 24576 16/30 Write block 65 len 4096 data: 8 17/30 Read block 407 len 12288 18/30 Read block 278 len 36864 19/30 Write block 519 len 24576 data: 13 15 14 14 12 10 20/30 Read block 274 len 20480 21/30 Read block 23 len 12288 22/30 Read block 372 len 8192 23/30 Read block 90 len 36864 24/30 Write block 444 len 16384 data: 10 10 11 9 25/30 Read block 306 len 12288 26/30 Flush 27/30 Write block 102 len 20480 data: 8 9 9 10 11 28/30 Write block 19 len 24576 data: 7 6 7 7 9 9 29/30 Write block 393 len 20480 data: 10 9 9 9 10 30/30 Read block 338 len 20480 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:36.728348195Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:36.728549009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1643,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 198 198 198 T F T <--- 1 020-039 111 111 111 198 198 198 T F T <--- 2 040-059 111 111 111 198 198 198 F F F 3 060-079 112 105 112 201 186 201 F F F <--- 4 080-099 112 101 112 201 179 201 F F F <--- 5 100-119 112 108 112 201 191 201 T F T <--- 6 120-139 111 111 111 198 198 198 F F F 7 140-159 112 106 112 199 187 199 F F F <--- 8 160-179 111 111 111 198 198 198 F F F 9 180-199 111 111 111 198 198 198 F F F 10 200-219 110 110 110 196 196 196 F F F 11 220-239 109 109 109 194 194 194 F F F 12 240-259 112 108 112 201 191 201 F F F <--- 13 260-279 112 110 112 199 197 199 F F F <--- 14 280-299 111 111 111 198 198 198 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 110 110 110 196 196 196 F F F 17 340-359 110 110 110 196 196 196 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 104 104 104 185 185 185 T F T <--- 20 400-419 110 110 110 197 197 197 F F F 21 420-439 110 110 110 196 196 196 F F F 22 440-459 112 108 112 201 189 201 F F F <--- 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 112 110 112 201 196 201 F F F <--- 26 520-539 112 110 112 201 196 201 F F F <--- 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 112, Max flush: 201 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 113 113 113 202 202 202 F F F 2 040-059 111 111 111 198 198 198 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 112 112 112 201 201 201 F F F 5 100-119 113 113 113 202 202 202 F F F 6 120-139 111 111 111 198 198 198 F F F 7 140-159 112 112 112 199 199 199 F F F 8 160-179 111 111 111 198 198 198 F F F 9 180-199 111 111 111 198 198 198 F F F 10 200-219 110 110 110 196 196 196 F F F 11 220-239 109 109 109 194 194 194 F F F 12 240-259 112 112 112 201 201 201 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 101 101 101 179 179 179 F F F 16 320-339 110 110 110 196 196 196 F F F 17 340-359 110 110 110 196 196 196 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 113 113 113 202 202 202 F F F 20 400-419 110 110 110 197 197 197 F F F 21 420-439 110 110 110 196 196 196 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 112 112 112 201 201 201 F F F 26 520-539 112 112 112 201 201 201 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 113, Max flush: 202 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:50.99788359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.006325308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.006629159Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.006661075Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.006670064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.006682362Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Crucible 699ad47d-3c4d-4c5e-bccc-54104e9fb531 has session id: bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007060583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Upstairs opts: Upstairs UUID: 699ad47d-3c4d-4c5e-bccc-54104e9fb531, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007092459Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Crucible stats registered with UUID: 699ad47d-3c4d-4c5e-bccc-54104e9fb531","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007104258Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:51.007113916Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.00723985Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007272546Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007284494Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007436575Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007454932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"699ad47d-3c4d-4c5e-bccc-54104e9fb531 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.00747415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007483629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"0","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007493887Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"1","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007504016Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"2","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007514115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007524413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007591035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"ds_connection connected from Ok(127.0.0.1:35132)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007617731Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"ds_connection connected from Ok(127.0.0.1:44130)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007632799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"ds_connection connected from Ok(127.0.0.1:60515)","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.007695271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008221843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"0","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.00824933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"1","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008336718Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"2","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008607173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[0]R flush_numbers[0..12]: [202, 202, 198, 201, 201, 202, 198, 199, 198, 198, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008636219Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[0]R generation[0..12]: [113, 113, 111, 112, 112, 113, 111, 112, 111, 111, 110, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008647898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008659576Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[1]R flush_numbers[0..12]: [202, 202, 198, 201, 201, 202, 198, 199, 198, 198, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008670785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[1]R generation[0..12]: [113, 113, 111, 112, 112, 113, 111, 112, 111, 111, 110, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008681794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008693252Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[2]R flush_numbers[0..12]: [202, 202, 198, 201, 201, 202, 198, 199, 198, 198, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008704271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[2]R generation[0..12]: [113, 113, 111, 112, 112, 113, 111, 112, 111, 111, 110, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008717359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008728887Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008738796Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008749045Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008758804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008768602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"":"downstairs","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008778801Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.00878794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"0","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008798159Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"1","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008808377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"client":"2","":"downstairs","session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008818956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008828685Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"699ad47d-3c4d-4c5e-bccc-54104e9fb531 is now active with session: bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008839073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008848392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"session_id":"bcfc4d39-ac6b-4e27-a7ad-8a28f2feb023"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008857891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Activated sub_volume 699ad47d-3c4d-4c5e-bccc-54104e9fb531","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.00886633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.008874899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.009223414Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"task":"crutest"} 01/30 Write block 310 len 20480 data: 12 15 11 10 10 02/30 Read block 578 len 20480 03/30 Write block 156 len 36864 data: 9 11 13 12 11 10 11 8 8 04/30 Write block 195 len 40960 data: 4 3 3 3 3 3 4 6 6 8 05/30 Write block 54 len 20480 data: 8 8 5 5 5 06/30 Write block 306 len 32768 data: 12 12 12 15 13 16 12 11 07/30 Read block 81 len 28672 08/30 Read block 344 len 8192 09/30 Read block 195 len 40960 10/30 Write block 195 len 32768 data: 5 4 4 4 4 4 5 7 11/30 Read block 333 len 32768 12/30 Write block 391 len 4096 data: 13 13/30 Read block 351 len 20480 14/30 Read block 59 len 36864 15/30 Read block 128 len 32768 16/30 Flush 17/30 Flush 18/30 Write block 105 len 20480 data: 11 12 11 11 10 19/30 Flush 20/30 Write block 38 len 28672 data: 8 8 10 10 10 10 12 21/30 Read block 95 len 4096 22/30 Flush 23/30 Write block 50 len 24576 data: 11 13 12 10 9 9 24/30 Read block 265 len 16384 25/30 Read block 386 len 16384 26/30 Read block 107 len 12288 27/30 Read block 457 len 12288 28/30 Write block 86 len 20480 data: 8 9 9 7 8 29/30 Read block 486 len 24576 30/30 Write block 344 len 40960 data: 5 6 7 7 7 9 11 10 9 10 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T20:59:51.018612579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T20:59:51.018813533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1652,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 113 206 206 202 F F F <--- 2 040-059 114 114 111 206 206 198 T T F <--- 3 060-079 112 112 112 201 201 201 F F F 4 080-099 112 112 112 201 201 201 T T F <--- 5 100-119 114 114 113 205 205 202 F F F <--- 6 120-139 111 111 111 198 198 198 F F F 7 140-159 114 114 112 203 203 199 F F F <--- 8 160-179 114 114 111 203 203 198 F F F <--- 9 180-199 114 114 111 203 203 198 F F F <--- 10 200-219 114 114 110 203 203 196 F F F <--- 11 220-239 109 109 109 194 194 194 F F F 12 240-259 112 112 112 201 201 201 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 114 114 101 203 203 179 F F F <--- 16 320-339 110 110 110 196 196 196 F F F 17 340-359 110 110 110 196 196 196 T T F <--- 18 360-379 106 106 106 187 187 187 F F F 19 380-399 114 114 113 203 203 202 F F F <--- 20 400-419 110 110 110 197 197 197 F F F 21 420-439 110 110 110 196 196 196 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 112 112 112 201 201 201 F F F 26 520-539 112 112 112 201 201 201 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 114, Max flush: 206 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 115 115 115 207 207 207 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 114 114 114 205 205 205 F F F 6 120-139 111 111 111 198 198 198 F F F 7 140-159 114 114 114 203 203 203 F F F 8 160-179 114 114 114 203 203 203 F F F 9 180-199 114 114 114 203 203 203 F F F 10 200-219 114 114 114 203 203 203 F F F 11 220-239 109 109 109 194 194 194 F F F 12 240-259 112 112 112 201 201 201 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 114 114 114 203 203 203 F F F 16 320-339 110 110 110 196 196 196 F F F 17 340-359 115 115 115 207 207 207 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 114 114 114 203 203 203 F F F 20 400-419 110 110 110 197 197 197 F F F 21 420-439 110 110 110 196 196 196 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 112 112 112 201 201 201 F F F 26 520-539 112 112 112 201 201 201 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 115, Max flush: 207 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.276159105Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:05.283320179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.283374442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.290203059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.290470894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29050557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.290515179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.290527007Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Crucible 22472937-c59a-48f9-9735-5ca66ac61be2 has session id: ac70b2a5-25a9-418c-90bb-66d5a37245ce","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.290872392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Upstairs opts: Upstairs UUID: 22472937-c59a-48f9-9735-5ca66ac61be2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29089179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Crucible stats registered with UUID: 22472937-c59a-48f9-9735-5ca66ac61be2","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.290904438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:05.290913507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291035801Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291053899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291064048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291220167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291230886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"22472937-c59a-48f9-9735-5ca66ac61be2 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291254303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291263682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"0","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29127334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"1","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291282979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"2","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291292498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291302547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291312495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:51090)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291420161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:54794)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291474185Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:44366)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.291517739Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292087935Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"1","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292106943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"0","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292152657Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"2","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292412263Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[0]R flush_numbers[0..12]: [202, 206, 207, 201, 207, 205, 198, 203, 203, 203, 203, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29243742Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[0]R generation[0..12]: [113, 114, 115, 112, 115, 114, 111, 114, 114, 114, 114, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292448918Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292460037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[1]R flush_numbers[0..12]: [202, 206, 207, 201, 207, 205, 198, 203, 203, 203, 203, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292470726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[1]R generation[0..12]: [113, 114, 115, 112, 115, 114, 111, 114, 114, 114, 114, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292481464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292492333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[2]R flush_numbers[0..12]: [202, 206, 207, 201, 207, 205, 198, 203, 203, 203, 203, 194]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292502861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[2]R generation[0..12]: [113, 114, 115, 112, 115, 114, 111, 114, 114, 114, 114, 109]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29251345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292524379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292533747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292543696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292554435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292563804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"":"downstairs","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292573572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292582351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"0","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29259206Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"1","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292601629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"client":"2","":"downstairs","session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292611697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292620966Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"22472937-c59a-48f9-9735-5ca66ac61be2 is now active with session: ac70b2a5-25a9-418c-90bb-66d5a37245ce","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292630915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292639744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"session_id":"ac70b2a5-25a9-418c-90bb-66d5a37245ce"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292648963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Activated sub_volume 22472937-c59a-48f9-9735-5ca66ac61be2","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292656962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.29266511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.292990548Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} 01/30 Write block 341 len 24576 data: 8 6 6 6 7 8 02/30 Flush 03/30 Write block 303 len 36864 data: 7 9 9 13 13 13 16 14 17 04/30 Write block 144 len 4096 data: 6 05/30 Read block 333 len 4096 06/30 Write block 405 len 24576 data: 13 13 14 12 11 10 07/30 Read block 446 len 12288 08/30 Flush 09/30 Write block 349 len 28672 data: 10 12 11 10 11 10 9 10/30 Flush 11/30 Flush 12/30 Write block 332 len 8192 data: 11 10 13/30 Read block 407 len 24576 14/30 Read block 581 len 16384 15/30 Write block 163 len 40960 data: 9 9 7 7 8 9 8 7 11 8 16/30 Read block 433 len 28672 17/30 Flush 18/30 Write block 162 len 28672 data: 12 10 10 8 8 9 10 19/30 Write block 233 len 16384 data: 5 3 8 7 20/30 Read block 288 len 8192 21/30 Read block 260 len 12288 22/30 Write block 390 len 8192 data: 13 14 23/30 Read block 553 len 20480 24/30 Read block 240 len 36864 25/30 Flush 26/30 Write block 233 len 12288 data: 6 4 9 27/30 Write block 231 len 32768 data: 5 5 7 5 10 8 8 9 28/30 Write block 308 len 4096 data: 14 29/30 Write block 211 len 32768 data: 3 4 5 4 6 7 9 13 30/30 Read block 151 len 12288 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 0 28 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:05.30217695Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:05.302371555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1661,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 115 115 115 207 207 207 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 114 114 114 205 205 205 F F F 6 120-139 111 111 111 198 198 198 F F F 7 140-159 114 116 116 203 209 209 F F F <--- 8 160-179 114 116 116 203 213 213 F F F <--- 9 180-199 114 114 114 203 203 203 F F F 10 200-219 114 114 114 203 203 203 F T T <--- 11 220-239 109 116 116 194 213 213 F T T <--- 12 240-259 112 112 112 201 201 201 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 114 116 116 203 209 209 F T T <--- 16 320-339 110 116 116 196 212 212 F F F <--- 17 340-359 116 116 116 208 210 210 F F F <--- 18 360-379 106 106 106 187 187 187 F F F 19 380-399 114 116 116 203 213 213 F F F <--- 20 400-419 110 116 116 197 209 209 F F F <--- 21 420-439 110 110 110 196 196 196 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 112 112 112 201 201 201 F F F 26 520-539 112 112 112 201 201 201 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 116, Max flush: 213 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 115 115 115 207 207 207 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 114 114 114 205 205 205 F F F 6 120-139 111 111 111 198 198 198 F F F 7 140-159 116 116 116 209 209 209 F F F 8 160-179 116 116 116 213 213 213 F F F 9 180-199 114 114 114 203 203 203 F F F 10 200-219 117 117 117 214 214 214 F F F 11 220-239 117 117 117 214 214 214 F F F 12 240-259 112 112 112 201 201 201 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 116 116 116 212 212 212 F F F 17 340-359 116 116 116 210 210 210 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 116 116 116 213 213 213 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 110 110 110 196 196 196 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 112 112 112 201 201 201 F F F 26 520-539 112 112 112 201 201 201 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 117, Max flush: 214 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.560681093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569200441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.56951356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569546216Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569557415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569569963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Crucible cc7a396f-d5cc-4445-b93f-3123c52d7499 has session id: acf4a234-67cb-4a94-9a8d-21bfd5a6f76f","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569920888Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Upstairs opts: Upstairs UUID: cc7a396f-d5cc-4445-b93f-3123c52d7499, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569952833Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Crucible stats registered with UUID: cc7a396f-d5cc-4445-b93f-3123c52d7499","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.569964922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:19.56998077Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570145598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570165176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570176095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570388887Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570407785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"cc7a396f-d5cc-4445-b93f-3123c52d7499 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570438101Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570447999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"0","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570458358Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"1","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570471636Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"2","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570481915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570492464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570503122Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62819)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570546987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58447)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570558695Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42414)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.570686049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571211581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"1","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571305898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"0","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571349493Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"2","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571644415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[0]R flush_numbers[0..12]: [202, 206, 207, 201, 207, 205, 198, 209, 213, 203, 214, 214]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571671931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[0]R generation[0..12]: [113, 114, 115, 112, 115, 114, 111, 116, 116, 114, 117, 117]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571684119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571695808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[1]R flush_numbers[0..12]: [202, 206, 207, 201, 207, 205, 198, 209, 213, 203, 214, 214]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571709286Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[1]R generation[0..12]: [113, 114, 115, 112, 115, 114, 111, 116, 116, 114, 117, 117]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571720645Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571732263Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[2]R flush_numbers[0..12]: [202, 206, 207, 201, 207, 205, 198, 209, 213, 203, 214, 214]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571743422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[2]R generation[0..12]: [113, 114, 115, 112, 115, 114, 111, 116, 116, 114, 117, 117]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.57175471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571766679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571776827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571787386Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571797565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571807664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"":"downstairs","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571818092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571827501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"0","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.57183762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"1","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571847828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"client":"2","":"downstairs","session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571858347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571869266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"cc7a396f-d5cc-4445-b93f-3123c52d7499 is now active with session: acf4a234-67cb-4a94-9a8d-21bfd5a6f76f","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571880044Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571889253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"session_id":"acf4a234-67cb-4a94-9a8d-21bfd5a6f76f"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571898912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Activated sub_volume cc7a396f-d5cc-4445-b93f-3123c52d7499","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571907471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.571916119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.572284732Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"task":"crutest"} 01/30 Write block 144 len 40960 data: 7 6 7 6 7 8 7 6 7 6 02/30 Write block 55 len 20480 data: 10 6 6 6 6 03/30 Write block 584 len 20480 data: 7 8 8 7 7 04/30 Write block 512 len 36864 data: 15 13 11 8 10 15 12 14 16 05/30 Flush 06/30 Write block 191 len 40960 data: 10 11 8 4 6 5 5 5 5 5 07/30 Read block 259 len 16384 08/30 Write block 105 len 8192 data: 12 13 09/30 Read block 354 len 8192 10/30 Write block 431 len 28672 data: 9 9 9 11 9 10 11 11/30 Read block 506 len 4096 12/30 Read block 113 len 12288 13/30 Flush 14/30 Write block 365 len 4096 data: 14 15/30 Write block 159 len 12288 data: 13 12 11 16/30 Read block 393 len 16384 17/30 Read block 112 len 4096 18/30 Read block 221 len 8192 19/30 Write block 113 len 40960 data: 9 10 11 8 8 7 7 7 10 9 20/30 Read block 148 len 40960 21/30 Write block 252 len 12288 data: 9 11 7 22/30 Read block 475 len 16384 23/30 Read block 158 len 20480 24/30 Read block 512 len 24576 25/30 Write block 177 len 8192 data: 11 10 26/30 Read block 421 len 12288 27/30 Read block 490 len 28672 28/30 Read block 331 len 40960 29/30 Write block 222 len 32768 data: 15 14 13 14 12 12 9 5 30/30 Read block 16 len 24576 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Write 5 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:19.581638042Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:19.581849115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1670,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 118 118 115 215 215 207 F F F <--- 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 118 118 114 216 216 205 T T F <--- 6 120-139 111 111 111 198 198 198 T T F <--- 7 140-159 118 118 116 215 215 209 T T F <--- 8 160-179 116 116 116 213 213 213 T T F <--- 9 180-199 118 118 114 216 216 203 F F F <--- 10 200-219 118 118 117 216 216 214 F F F <--- 11 220-239 117 117 117 214 214 214 T T F <--- 12 240-259 112 112 112 201 201 201 T T F <--- 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 116 116 116 212 212 212 F F F 17 340-359 116 116 116 210 210 210 F F F 18 360-379 106 106 106 187 187 187 T T F <--- 19 380-399 116 116 116 213 213 213 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 110 216 216 196 F F F <--- 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 118 118 112 215 215 201 F F F <--- 26 520-539 118 118 112 215 215 201 F F F <--- 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 118 118 106 215 215 187 F F F <--- Max gen: 118, Max flush: 216 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 118 118 118 215 215 215 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 119 119 119 217 217 217 F F F 6 120-139 119 119 119 217 217 217 F F F 7 140-159 119 119 119 217 217 217 F F F 8 160-179 119 119 119 217 217 217 F F F 9 180-199 118 118 118 216 216 216 F F F 10 200-219 118 118 118 216 216 216 F F F 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 116 116 116 212 212 212 F F F 17 340-359 116 116 116 210 210 210 F F F 18 360-379 119 119 119 217 217 217 F F F 19 380-399 116 116 116 213 213 213 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 118 118 118 215 215 215 F F F 26 520-539 118 118 118 215 215 215 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F F F 29 580-599 118 118 118 215 215 215 F F F Max gen: 119, Max flush: 217 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.854823545Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:33.862062229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.862115102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.86908487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869352246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869382282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869390871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869402699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Crucible 9e917401-530a-40d6-944c-7fe7641eb613 has session id: 80df088a-d054-4627-9dcc-5172fe34f538","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.86978516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Upstairs opts: Upstairs UUID: 9e917401-530a-40d6-944c-7fe7641eb613, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869818246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Crucible stats registered with UUID: 9e917401-530a-40d6-944c-7fe7641eb613","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869830114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:33.869839613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869957148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869974295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.869984504Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870155532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.87017244Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"9e917401-530a-40d6-944c-7fe7641eb613 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870199646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870209055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"0","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870218874Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"1","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870228612Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"2","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870238301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.87024817Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870258039Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"ds_connection connected from Ok(127.0.0.1:64998)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870379813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"ds_connection connected from Ok(127.0.0.1:45231)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870437895Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"ds_connection connected from Ok(127.0.0.1:33348)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.870460452Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871107879Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"1","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871124277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"0","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871135445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"2","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871459343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[0]R flush_numbers[0..12]: [202, 206, 215, 201, 207, 217, 217, 217, 217, 216, 216, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.87148507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[0]R generation[0..12]: [113, 114, 118, 112, 115, 119, 119, 119, 119, 118, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871496498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871507787Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[1]R flush_numbers[0..12]: [202, 206, 215, 201, 207, 217, 217, 217, 217, 216, 216, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871518626Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[1]R generation[0..12]: [113, 114, 118, 112, 115, 119, 119, 119, 119, 118, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871529124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871540073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[2]R flush_numbers[0..12]: [202, 206, 215, 201, 207, 217, 217, 217, 217, 216, 216, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871550821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[2]R generation[0..12]: [113, 114, 118, 112, 115, 119, 119, 119, 119, 118, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.87156334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871574388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871583837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871593876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871603295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871612773Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"":"downstairs","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871622602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871631441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"0","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.8716413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"1","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871651378Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"client":"2","":"downstairs","session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871661597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871670866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"9e917401-530a-40d6-944c-7fe7641eb613 is now active with session: 80df088a-d054-4627-9dcc-5172fe34f538","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871680955Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871689933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"session_id":"80df088a-d054-4627-9dcc-5172fe34f538"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871699132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Activated sub_volume 9e917401-530a-40d6-944c-7fe7641eb613","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.871707171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.87171688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.872046717Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} 01/30 Flush 02/30 Write block 329 len 16384 data: 13 10 12 12 03/30 Write block 344 len 36864 data: 7 8 9 8 8 11 13 12 11 04/30 Read block 410 len 20480 05/30 Read block 570 len 20480 06/30 Read block 194 len 4096 07/30 Read block 208 len 4096 08/30 Write block 164 len 28672 data: 11 9 9 10 11 9 8 09/30 Write block 191 len 12288 data: 11 12 9 10/30 Read block 322 len 36864 11/30 Read block 503 len 4096 12/30 Write block 103 len 20480 data: 10 10 13 14 12 13/30 Read block 184 len 8192 14/30 Write block 128 len 40960 data: 6 7 7 7 7 9 8 7 7 6 15/30 Read block 386 len 4096 16/30 Read block 299 len 12288 17/30 Write block 354 len 36864 data: 11 10 7 6 9 10 11 10 15 18/30 Read block 99 len 36864 19/30 Read block 71 len 32768 20/30 Write block 152 len 24576 data: 8 7 10 8 10 12 21/30 Write block 122 len 36864 data: 10 10 11 11 8 6 7 8 8 22/30 Write block 50 len 16384 data: 12 14 13 11 23/30 Read block 169 len 4096 24/30 Write block 333 len 4096 data: 11 25/30 Read block 18 len 28672 26/30 Flush 27/30 Write block 568 len 8192 data: 8 8 28/30 Read block 515 len 32768 29/30 Read block 219 len 16384 30/30 Write block 54 len 24576 data: 10 11 7 7 7 7 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:33.881098277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:33.881285312Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1679,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 118 120 120 215 219 219 F T T <--- 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 119 120 120 217 219 219 F F F <--- 6 120-139 119 120 120 217 219 219 F F F <--- 7 140-159 119 120 120 217 219 219 F F F <--- 8 160-179 119 120 120 217 219 219 F F F <--- 9 180-199 118 120 120 216 219 219 F F F <--- 10 200-219 118 118 118 216 216 216 F F F 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 116 120 120 212 219 219 F F F <--- 17 340-359 116 120 120 210 219 219 F F F <--- 18 360-379 119 120 120 217 219 219 F F F <--- 19 380-399 116 116 116 213 213 213 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 118 118 118 215 215 215 F F F 26 520-539 118 118 118 215 215 215 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 108 108 108 193 193 193 F T T <--- 29 580-599 118 118 118 215 215 215 F F F Max gen: 120, Max flush: 219 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 202 202 202 F F F 1 020-039 114 114 114 206 206 206 F F F 2 040-059 121 121 121 220 220 220 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 120 120 120 219 219 219 F F F 6 120-139 120 120 120 219 219 219 F F F 7 140-159 120 120 120 219 219 219 F F F 8 160-179 120 120 120 219 219 219 F F F 9 180-199 120 120 120 219 219 219 F F F 10 200-219 118 118 118 216 216 216 F F F 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 112 112 112 199 199 199 F F F 14 280-299 111 111 111 198 198 198 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 120 120 120 219 219 219 F F F 17 340-359 120 120 120 219 219 219 F F F 18 360-379 120 120 120 219 219 219 F F F 19 380-399 116 116 116 213 213 213 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 110 110 110 196 196 196 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 118 118 118 215 215 215 F F F 26 520-539 118 118 118 215 215 215 F F F 27 540-559 108 108 108 193 193 193 F F F 28 560-579 121 121 121 220 220 220 F F F 29 580-599 118 118 118 215 215 215 F F F Max gen: 121, Max flush: 220 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.152046189Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.160682082Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.160992822Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161026848Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161036077Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161048485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Crucible ad571d2b-0d9f-4328-97a1-e73bc2f1c9cd has session id: 9dea2af8-2c55-4210-a2f1-9f3a502eecd7","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161360635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Upstairs opts: Upstairs UUID: ad571d2b-0d9f-4328-97a1-e73bc2f1c9cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161463651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Crucible stats registered with UUID: ad571d2b-0d9f-4328-97a1-e73bc2f1c9cd","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.16147515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:48.161485199Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.16163063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161648567Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161661596Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161824165Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161842602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"ad571d2b-0d9f-4328-97a1-e73bc2f1c9cd active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161872738Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161882757Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"0","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161893306Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"1","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161903724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"2","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161913893Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161924572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.16193526Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62292)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161972216Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39011)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161984044Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61270)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.161995083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.162658447Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"1","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.162677294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"0","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.162719989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"2","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163025509Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[0]R flush_numbers[0..12]: [202, 206, 220, 201, 207, 219, 219, 219, 219, 219, 216, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163058235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[0]R generation[0..12]: [113, 114, 121, 112, 115, 120, 120, 120, 120, 120, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163070444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163082382Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[1]R flush_numbers[0..12]: [202, 206, 220, 201, 207, 219, 219, 219, 219, 219, 216, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163093911Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[1]R generation[0..12]: [113, 114, 121, 112, 115, 120, 120, 120, 120, 120, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163105259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163116868Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[2]R flush_numbers[0..12]: [202, 206, 220, 201, 207, 219, 219, 219, 219, 219, 216, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163128196Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[2]R generation[0..12]: [113, 114, 121, 112, 115, 120, 120, 120, 120, 120, 118, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163139625Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163151273Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163161422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.16317208Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163182289Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163193798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"":"downstairs","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163204356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163213905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"0","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163224384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"1","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163234802Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"client":"2","":"downstairs","session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163245531Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.16325542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"ad571d2b-0d9f-4328-97a1-e73bc2f1c9cd is now active with session: 9dea2af8-2c55-4210-a2f1-9f3a502eecd7","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163266048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163280206Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"session_id":"9dea2af8-2c55-4210-a2f1-9f3a502eecd7"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163290115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Activated sub_volume ad571d2b-0d9f-4328-97a1-e73bc2f1c9cd","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163298744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163307443Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.163540953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"task":"crutest"} 01/30 Flush 02/30 Write block 574 len 16384 data: 6 8 8 9 03/30 Write block 294 len 12288 data: 9 9 8 04/30 Write block 207 len 16384 data: 10 7 8 5 05/30 Read block 51 len 28672 06/30 Write block 527 len 4096 data: 13 07/30 Read block 289 len 8192 08/30 Write block 7 len 20480 data: 7 7 7 6 9 09/30 Write block 461 len 32768 data: 10 10 11 14 12 9 12 13 10/30 Write block 182 len 20480 data: 9 12 11 10 13 11/30 Flush 12/30 Write block 276 len 20480 data: 12 10 9 8 10 13/30 Read block 69 len 16384 14/30 Write block 378 len 32768 data: 6 8 8 11 12 10 10 11 15/30 Write block 360 len 20480 data: 12 11 16 16 18 16/30 Read block 218 len 8192 17/30 Write block 165 len 28672 data: 10 10 11 12 10 9 12 18/30 Write block 501 len 20480 data: 8 8 8 9 7 19/30 Read block 228 len 12288 20/30 Read block 557 len 32768 21/30 Read block 529 len 40960 22/30 Write block 535 len 40960 data: 10 11 12 13 11 13 14 15 11 8 23/30 Write block 378 len 24576 data: 7 9 9 12 13 11 24/30 Read block 31 len 8192 25/30 Read block 73 len 12288 26/30 Read block 314 len 16384 27/30 Write block 15 len 28672 data: 7 8 8 7 8 7 8 28/30 Write block 560 len 32768 data: 13 13 14 10 13 12 12 9 29/30 Read block 473 len 32768 30/30 Write block 589 len 36864 data: 9 11 9 12 13 10 10 8 6 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Done Done false 1002 Acked Write 3 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:00:48.172340175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:00:48.172570295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1688,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 113 222 222 202 T T F <--- 1 020-039 114 114 114 206 206 206 T T F <--- 2 040-059 121 121 121 220 220 220 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 120 120 120 219 219 219 F F F 6 120-139 120 120 120 219 219 219 F F F 7 140-159 120 120 120 219 219 219 F F F 8 160-179 120 120 120 219 219 219 T T F <--- 9 180-199 122 122 120 222 222 219 F F F <--- 10 200-219 122 122 118 222 222 216 F F F <--- 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 112 112 112 199 199 199 T T F <--- 14 280-299 122 122 111 222 222 198 T T F <--- 15 300-319 117 117 117 214 214 214 F F F 16 320-339 120 120 120 219 219 219 F F F 17 340-359 120 120 120 219 219 219 F F F 18 360-379 120 120 120 219 219 219 T T F <--- 19 380-399 116 116 116 213 213 213 T T F <--- 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 122 122 110 222 222 196 F F F <--- 24 480-499 111 111 111 198 198 198 F F F 25 500-519 118 118 118 215 215 215 T T F <--- 26 520-539 122 122 118 222 222 215 T T F <--- 27 540-559 108 108 108 193 193 193 T T F <--- 28 560-579 122 122 121 222 222 220 T T T <--- 29 580-599 118 118 118 215 215 215 T T F <--- Max gen: 122, Max flush: 222 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 223 223 223 F F F 1 020-039 123 123 123 223 223 223 F F F 2 040-059 121 121 121 220 220 220 F F F 3 060-079 112 112 112 201 201 201 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 120 120 120 219 219 219 F F F 6 120-139 120 120 120 219 219 219 F F F 7 140-159 120 120 120 219 219 219 F F F 8 160-179 123 123 123 223 223 223 F F F 9 180-199 122 122 122 222 222 222 F F F 10 200-219 122 122 122 222 222 222 F F F 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 123 123 123 223 223 223 F F F 14 280-299 123 123 123 223 223 223 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 120 120 120 219 219 219 F F F 17 340-359 120 120 120 219 219 219 F F F 18 360-379 123 123 123 223 223 223 F F F 19 380-399 123 123 123 223 223 223 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 122 122 122 222 222 222 F F F 24 480-499 111 111 111 198 198 198 F F F 25 500-519 123 123 123 223 223 223 F F F 26 520-539 123 123 123 223 223 223 F F F 27 540-559 123 123 123 223 223 223 F F F 28 560-579 123 123 123 223 223 223 F F F 29 580-599 123 123 123 223 223 223 F F F Max gen: 123, Max flush: 223 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.449250876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:02.456762495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.456819447Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.463858897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464126662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464157078Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464165567Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464177176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Crucible b66cf1ce-824d-46d6-ac0f-01912d03e818 has session id: 6b60d195-4eb0-435e-87ed-54a3f7c87e2c","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464594092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Upstairs opts: Upstairs UUID: b66cf1ce-824d-46d6-ac0f-01912d03e818, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.46460975Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Crucible stats registered with UUID: b66cf1ce-824d-46d6-ac0f-01912d03e818","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464619978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:02.464629167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464754241Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464770729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464780968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464974053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.464987841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"b66cf1ce-824d-46d6-ac0f-01912d03e818 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465023786Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465034805Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"0","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465044763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"1","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465054622Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"2","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465064391Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.46507465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465084868Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40134)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465159609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64450)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465171037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47646)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465181566Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465857948Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"1","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465884325Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"0","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.465896503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"2","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.46622923Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[0]R flush_numbers[0..12]: [223, 223, 220, 201, 207, 219, 219, 219, 223, 222, 222, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466265256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[0]R generation[0..12]: [123, 123, 121, 112, 115, 120, 120, 120, 123, 122, 122, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466276854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466289902Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[1]R flush_numbers[0..12]: [223, 223, 220, 201, 207, 219, 219, 219, 223, 222, 222, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466300791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[1]R generation[0..12]: [123, 123, 121, 112, 115, 120, 120, 120, 123, 122, 122, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.46631146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466322798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[2]R flush_numbers[0..12]: [223, 223, 220, 201, 207, 219, 219, 219, 223, 222, 222, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466333697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[2]R generation[0..12]: [123, 123, 121, 112, 115, 120, 120, 120, 123, 122, 122, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466344365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466355454Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466365113Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466375141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.46638468Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466394429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"":"downstairs","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466404508Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466413516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"0","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466423465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"1","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466434394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"client":"2","":"downstairs","session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466444392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466453751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"b66cf1ce-824d-46d6-ac0f-01912d03e818 is now active with session: 6b60d195-4eb0-435e-87ed-54a3f7c87e2c","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.46646396Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466472919Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"session_id":"6b60d195-4eb0-435e-87ed-54a3f7c87e2c"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466482228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Activated sub_volume b66cf1ce-824d-46d6-ac0f-01912d03e818","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466490367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466498615Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.466841841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} 01/30 Write block 580 len 36864 data: 8 5 5 6 8 9 9 8 8 02/30 Read block 456 len 16384 03/30 Flush 04/30 Write block 320 len 40960 data: 6 6 5 7 6 8 10 12 15 14 05/30 Read block 37 len 40960 06/30 Read block 150 len 36864 07/30 Read block 260 len 28672 08/30 Read block 436 len 4096 09/30 Write block 18 len 16384 data: 8 9 8 9 10/30 Write block 124 len 28672 data: 12 12 9 7 8 9 9 11/30 Write block 1 len 8192 data: 5 5 12/30 Read block 438 len 40960 13/30 Flush 14/30 Write block 60 len 28672 data: 7 11 12 11 9 9 9 15/30 Write block 536 len 12288 data: 12 13 14 16/30 Flush 17/30 Write block 485 len 20480 data: 7 6 3 5 6 18/30 Read block 575 len 16384 19/30 Write block 66 len 12288 data: 10 10 9 20/30 Read block 524 len 20480 21/30 Read block 348 len 24576 22/30 Write block 540 len 40960 data: 14 15 16 12 9 8 9 10 10 10 23/30 Write block 564 len 28672 data: 14 13 13 10 9 9 8 24/30 Read block 563 len 28672 25/30 Write block 565 len 4096 data: 14 26/30 Flush 27/30 Write block 472 len 4096 data: 16 28/30 Write block 106 len 32768 data: 15 13 12 11 9 7 7 10 29/30 Write block 354 len 36864 data: 12 11 8 7 10 11 13 12 17 30/30 Write block 199 len 16384 data: 6 6 6 8 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 3 3 33 Done 0 24 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:02.476005716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:02.476209559Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1697,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 124 124 223 225 225 F F F <--- 1 020-039 123 124 124 223 225 225 F F F <--- 2 040-059 121 121 121 220 220 220 F F F 3 060-079 112 124 124 201 227 227 F F F <--- 4 080-099 115 115 115 207 207 207 F F F 5 100-119 120 120 120 219 219 219 F T T <--- 6 120-139 120 124 124 219 225 225 F F F <--- 7 140-159 120 120 120 219 219 219 F F F 8 160-179 123 123 123 223 223 223 F F F 9 180-199 122 122 122 222 222 222 F T T <--- 10 200-219 122 122 122 222 222 222 F T T <--- 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 123 123 123 223 223 223 F F F 14 280-299 123 123 123 223 223 223 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 120 124 124 219 225 225 F F F <--- 17 340-359 120 120 120 219 219 219 F T T <--- 18 360-379 123 123 123 223 223 223 F T T <--- 19 380-399 123 123 123 223 223 223 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 122 122 122 222 222 222 F T T <--- 24 480-499 111 124 124 198 227 227 F F F <--- 25 500-519 123 123 123 223 223 223 F F F 26 520-539 123 124 124 223 226 226 F F F <--- 27 540-559 123 124 124 223 227 227 F F F <--- 28 560-579 123 124 124 223 227 227 F F F <--- 29 580-599 124 124 124 224 224 224 F F F Max gen: 124, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 225 225 225 F F F 1 020-039 124 124 124 225 225 225 F F F 2 040-059 121 121 121 220 220 220 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 115 115 115 207 207 207 F F F 5 100-119 125 125 125 228 228 228 F F F 6 120-139 124 124 124 225 225 225 F F F 7 140-159 120 120 120 219 219 219 F F F 8 160-179 123 123 123 223 223 223 F F F 9 180-199 125 125 125 228 228 228 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 119 119 119 217 217 217 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 123 123 123 223 223 223 F F F 14 280-299 123 123 123 223 223 223 F F F 15 300-319 117 117 117 214 214 214 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 125 125 125 228 228 228 F F F 18 360-379 125 125 125 228 228 228 F F F 19 380-399 123 123 123 223 223 223 F F F 20 400-419 116 116 116 209 209 209 F F F 21 420-439 118 118 118 216 216 216 F F F 22 440-459 112 112 112 201 201 201 F F F 23 460-479 125 125 125 228 228 228 F F F 24 480-499 124 124 124 227 227 227 F F F 25 500-519 123 123 123 223 223 223 F F F 26 520-539 124 124 124 226 226 226 F F F 27 540-559 124 124 124 227 227 227 F F F 28 560-579 124 124 124 227 227 227 F F F 29 580-599 124 124 124 224 224 224 F F F Max gen: 125, Max flush: 228 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.755777197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.76465076Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.76495512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.764985706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765000834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765013993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Crucible 7e0383f9-79bf-4543-a09b-1519c6a1001b has session id: cd94f316-09c9-491e-afba-bbde6ed491f6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765459085Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Upstairs opts: Upstairs UUID: 7e0383f9-79bf-4543-a09b-1519c6a1001b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765492281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Crucible stats registered with UUID: 7e0383f9-79bf-4543-a09b-1519c6a1001b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765504449Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:16.765514898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765667068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765685806Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765697034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765860283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765870972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"7e0383f9-79bf-4543-a09b-1519c6a1001b active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765910927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765921635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"0","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765932504Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"1","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765943323Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"2","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765954171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.76596543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765976708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43806)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.765998795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:64409)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.766012114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65256)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.766136648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.766749648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"0","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.766771295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"1","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.766856744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"2","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767128399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 220, 227, 207, 228, 225, 219, 223, 228, 228, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767150156Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[0]R generation[0..12]: [124, 124, 121, 124, 115, 125, 124, 120, 123, 125, 125, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767162215Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767174573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 220, 227, 207, 228, 225, 219, 223, 228, 228, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767186522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[1]R generation[0..12]: [124, 124, 121, 124, 115, 125, 124, 120, 123, 125, 125, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.76719824Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767210549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 220, 227, 207, 228, 225, 219, 223, 228, 228, 217]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767222457Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[2]R generation[0..12]: [124, 124, 121, 124, 115, 125, 124, 120, 123, 125, 125, 119]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767236095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767248354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767258962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767269891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.76728045Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767290928Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"":"downstairs","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767301787Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767311516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"0","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767322214Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"1","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767332983Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"client":"2","":"downstairs","session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767344011Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.76735429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"7e0383f9-79bf-4543-a09b-1519c6a1001b is now active with session: cd94f316-09c9-491e-afba-bbde6ed491f6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767365389Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767375237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"session_id":"cd94f316-09c9-491e-afba-bbde6ed491f6"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2026-05-14T21:01:16.767390555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Activated sub_volume 7e0383f9-79bf-4543-a09b-1519c6a1001b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767399674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767409943Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.767674119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"task":"crutest"} 01/30 Write block 136 len 28672 data: 8 7 6 8 6 5 6 02/30 Read block 160 len 20480 03/30 Write block 487 len 24576 data: 4 6 7 10 11 9 04/30 Write block 18 len 12288 data: 9 10 9 05/30 Read block 479 len 20480 06/30 Write block 487 len 28672 data: 5 7 8 11 12 10 11 07/30 Write block 397 len 16384 data: 11 8 5 4 08/30 Write block 457 len 12288 data: 7 9 10 09/30 Read block 486 len 16384 10/30 Read block 523 len 8192 11/30 Write block 142 len 8192 data: 7 5 12/30 Read block 184 len 4096 13/30 Write block 314 len 16384 data: 11 10 9 10 14/30 Write block 97 len 24576 data: 11 11 13 11 9 9 15/30 Write block 358 len 40960 data: 11 12 14 13 18 17 19 15 16 15 16/30 Write block 225 len 24576 data: 15 13 13 10 6 5 17/30 Write block 434 len 20480 data: 12 10 11 12 11 18/30 Read block 143 len 32768 19/30 Read block 176 len 16384 20/30 Read block 543 len 8192 21/30 Read block 376 len 28672 22/30 Read block 211 len 12288 23/30 Read block 422 len 20480 24/30 Write block 357 len 16384 data: 8 12 13 15 25/30 Flush 26/30 Flush 27/30 Write block 580 len 4096 data: 9 28/30 Read block 61 len 36864 29/30 Read block 536 len 20480 30/30 Read block 284 len 20480 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 0 29 Done 29 2 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:16.776817646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:16.777038517Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1706,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 124 126 229 225 229 F F F <--- 1 020-039 126 124 126 229 225 229 F F F <--- 2 040-059 121 121 121 220 220 220 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 126 115 126 229 207 229 F F F <--- 5 100-119 126 125 126 229 228 229 F F F <--- 6 120-139 126 124 126 229 225 229 F T F <--- 7 140-159 126 120 126 229 219 229 F T F <--- 8 160-179 123 123 123 223 223 223 F F F 9 180-199 125 125 125 228 228 228 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 126 119 126 229 217 229 F F F <--- 12 240-259 119 119 119 217 217 217 F F F 13 260-279 123 123 123 223 223 223 F F F 14 280-299 123 123 123 223 223 223 F F F 15 300-319 126 117 126 229 214 229 F F F <--- 16 320-339 124 124 124 225 225 225 F F F 17 340-359 126 125 126 229 228 229 F F F <--- 18 360-379 126 125 126 229 228 229 F F F <--- 19 380-399 126 123 126 229 223 229 F F F <--- 20 400-419 126 116 126 229 209 229 F F F <--- 21 420-439 126 118 126 229 216 229 F F F <--- 22 440-459 126 112 126 229 201 229 F F F <--- 23 460-479 125 125 125 228 228 228 F F F 24 480-499 126 124 126 229 227 229 F F F <--- 25 500-519 123 123 123 223 223 223 F F F 26 520-539 124 124 124 226 226 226 F F F 27 540-559 124 124 124 227 227 227 F F F 28 560-579 124 124 124 227 227 227 F F F 29 580-599 124 124 124 224 224 224 T F T <--- Max gen: 126, Max flush: 229 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 229 229 229 F F F 1 020-039 126 126 126 229 229 229 F F F 2 040-059 121 121 121 220 220 220 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 126 126 126 229 229 229 F F F 5 100-119 126 126 126 229 229 229 F F F 6 120-139 126 126 126 229 229 229 F F F 7 140-159 126 126 126 229 229 229 F F F 8 160-179 123 123 123 223 223 223 F F F 9 180-199 125 125 125 228 228 228 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 126 126 126 229 229 229 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 123 123 123 223 223 223 F F F 14 280-299 123 123 123 223 223 223 F F F 15 300-319 126 126 126 229 229 229 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 126 126 126 229 229 229 F F F 18 360-379 126 126 126 229 229 229 F F F 19 380-399 126 126 126 229 229 229 F F F 20 400-419 126 126 126 229 229 229 F F F 21 420-439 126 126 126 229 229 229 F F F 22 440-459 126 126 126 229 229 229 F F F 23 460-479 125 125 125 228 228 228 F F F 24 480-499 126 126 126 229 229 229 F F F 25 500-519 123 123 123 223 223 223 F F F 26 520-539 124 124 124 226 226 226 F F F 27 540-559 124 124 124 227 227 227 F F F 28 560-579 124 124 124 227 227 227 F F F 29 580-599 127 127 127 230 230 230 F F F Max gen: 127, Max flush: 230 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.059320154Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.067651166Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.067939419Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.067971345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.067980294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.067991952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Crucible 9c3504ce-4dc7-40b2-982f-145ae14f647d has session id: 7908ff41-2327-417e-ad6d-06e3960f2e7e","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068288614Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Upstairs opts: Upstairs UUID: 9c3504ce-4dc7-40b2-982f-145ae14f647d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068305592Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Crucible stats registered with UUID: 9c3504ce-4dc7-40b2-982f-145ae14f647d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.0683159Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:31.068363594Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068530973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068561239Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068574617Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068747714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068760543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"9c3504ce-4dc7-40b2-982f-145ae14f647d active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068794998Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068804537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"0","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068814396Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"1","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068824305Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"2","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068834163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068844262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.06885698Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37448)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.06894047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36025)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.068966976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52751)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.06901332Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.06963573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"2","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.069677724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"1","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.069689613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"0","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070003262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[0]R flush_numbers[0..12]: [229, 229, 220, 227, 229, 229, 229, 229, 223, 228, 228, 229]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070050006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[0]R generation[0..12]: [126, 126, 121, 124, 126, 126, 126, 126, 123, 125, 125, 126]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070061635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070073023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[1]R flush_numbers[0..12]: [229, 229, 220, 227, 229, 229, 229, 229, 223, 228, 228, 229]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070084022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[1]R generation[0..12]: [126, 126, 121, 124, 126, 126, 126, 126, 123, 125, 125, 126]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.07009482Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070106209Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[2]R flush_numbers[0..12]: [229, 229, 220, 227, 229, 229, 229, 229, 223, 228, 228, 229]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070117157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[2]R generation[0..12]: [126, 126, 121, 124, 126, 126, 126, 126, 123, 125, 125, 126]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070128016Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070139215Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070149003Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070159112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070168971Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070179969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"":"downstairs","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070190548Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070199847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"0","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070210425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"1","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070220774Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"client":"2","":"downstairs","session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070231343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070240941Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"9c3504ce-4dc7-40b2-982f-145ae14f647d is now active with session: 7908ff41-2327-417e-ad6d-06e3960f2e7e","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.07025145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070260819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"session_id":"7908ff41-2327-417e-ad6d-06e3960f2e7e"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070270508Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Activated sub_volume 9c3504ce-4dc7-40b2-982f-145ae14f647d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070278916Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070287435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.070623812Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"task":"crutest"} 01/30 Write block 13 len 36864 data: 6 6 8 9 9 10 11 10 10 02/30 Write block 342 len 24576 data: 7 7 8 9 10 9 03/30 Write block 137 len 16384 data: 8 7 9 7 04/30 Write block 300 len 16384 data: 7 9 8 8 05/30 Read block 252 len 20480 06/30 Read block 30 len 40960 07/30 Read block 215 len 32768 08/30 Read block 234 len 36864 09/30 Write block 181 len 36864 data: 9 10 13 12 11 14 10 9 9 10/30 Read block 343 len 12288 11/30 Read block 564 len 36864 12/30 Write block 392 len 20480 data: 13 11 10 10 10 13/30 Read block 504 len 28672 14/30 Write block 581 len 16384 data: 6 6 7 9 15/30 Write block 575 len 36864 data: 9 9 10 8 7 10 7 7 8 16/30 Read block 187 len 28672 17/30 Read block 309 len 16384 18/30 Read block 270 len 40960 19/30 Flush 20/30 Flush 21/30 Write block 46 len 20480 data: 10 13 13 13 13 22/30 Write block 267 len 20480 data: 9 9 8 9 11 23/30 Write block 379 len 40960 data: 10 10 13 14 12 11 12 12 11 12 24/30 Read block 378 len 24576 25/30 Write block 177 len 28672 data: 12 11 8 9 10 11 14 26/30 Flush 27/30 Write block 275 len 8192 data: 13 13 28/30 Write block 6 len 20480 data: 7 8 8 8 7 29/30 Write block 473 len 8192 data: 14 9 30/30 Write block 501 len 24576 data: 9 9 9 10 8 10 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 5 Done Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 3 34 Done 28 1 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:31.079706577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:31.079908521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1715,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 126 128 231 229 231 T T T <--- 1 020-039 128 126 128 231 229 231 F T F <--- 2 040-059 128 121 128 233 220 233 F F F <--- 3 060-079 124 124 124 227 227 227 F F F 4 080-099 126 126 126 229 229 229 F F F 5 100-119 126 126 126 229 229 229 F F F 6 120-139 128 126 128 231 229 231 F F F <--- 7 140-159 128 126 128 231 229 231 F F F <--- 8 160-179 128 123 128 233 223 233 F F F <--- 9 180-199 128 125 128 233 228 233 F F F <--- 10 200-219 125 125 125 228 228 228 F F F 11 220-239 126 126 126 229 229 229 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 128 123 128 233 223 233 T F T <--- 14 280-299 123 123 123 223 223 223 F F F 15 300-319 128 126 128 231 229 231 F F F <--- 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 126 128 231 229 231 F F F <--- 18 360-379 128 126 128 233 229 233 F F F <--- 19 380-399 128 126 128 233 229 233 F F F <--- 20 400-419 126 126 126 229 229 229 F F F 21 420-439 126 126 126 229 229 229 F F F 22 440-459 126 126 126 229 229 229 F F F 23 460-479 125 125 125 228 228 228 T F T <--- 24 480-499 126 126 126 229 229 229 F F F 25 500-519 123 123 123 223 223 223 T F T <--- 26 520-539 124 124 124 226 226 226 F F F 27 540-559 124 124 124 227 227 227 F F F 28 560-579 128 124 128 231 227 231 F F F <--- 29 580-599 128 127 128 231 230 231 F F F <--- Max gen: 128, Max flush: 233 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 234 234 234 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 128 128 128 233 233 233 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 126 126 126 229 229 229 F F F 5 100-119 126 126 126 229 229 229 F F F 6 120-139 128 128 128 231 231 231 F F F 7 140-159 128 128 128 231 231 231 F F F 8 160-179 128 128 128 233 233 233 F F F 9 180-199 128 128 128 233 233 233 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 126 126 126 229 229 229 F F F 12 240-259 119 119 119 217 217 217 F F F 13 260-279 129 129 129 234 234 234 F F F 14 280-299 123 123 123 223 223 223 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 128 128 128 233 233 233 F F F 19 380-399 128 128 128 233 233 233 F F F 20 400-419 126 126 126 229 229 229 F F F 21 420-439 126 126 126 229 229 229 F F F 22 440-459 126 126 126 229 229 229 F F F 23 460-479 129 129 129 234 234 234 F F F 24 480-499 126 126 126 229 229 229 F F F 25 500-519 129 129 129 234 234 234 F F F 26 520-539 124 124 124 226 226 226 F F F 27 540-559 124 124 124 227 227 227 F F F 28 560-579 128 128 128 231 231 231 F F F 29 580-599 128 128 128 231 231 231 F F F Max gen: 129, Max flush: 234 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.358291392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.36681618Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367129189Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367157875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367169404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367181672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Crucible adc152a8-0bc8-4fd3-9fbf-9b0a19e10a3e has session id: 4f4e0b05-d661-4aec-b6c1-7f6c336a8643","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367564683Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Upstairs opts: Upstairs UUID: adc152a8-0bc8-4fd3-9fbf-9b0a19e10a3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.36758602Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Crucible stats registered with UUID: adc152a8-0bc8-4fd3-9fbf-9b0a19e10a3e","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367597808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:45.367607427Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367747299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367779345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367790694Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367928626Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367946533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"adc152a8-0bc8-4fd3-9fbf-9b0a19e10a3e active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367981549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.367991478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"0","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368010495Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"1","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368022774Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"2","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368032942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368043331Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.36805375Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"ds_connection connected from Ok(127.0.0.1:47568)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368147837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"ds_connection connected from Ok(127.0.0.1:51759)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368175144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"ds_connection connected from Ok(127.0.0.1:55765)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368275381Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368920297Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"0","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368944334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"1","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.368967541Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"2","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369314426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[0]R flush_numbers[0..12]: [234, 231, 233, 227, 229, 229, 231, 231, 233, 233, 228, 229]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369341973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[0]R generation[0..12]: [129, 128, 128, 124, 126, 126, 128, 128, 128, 128, 125, 126]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369354061Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.36936592Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[1]R flush_numbers[0..12]: [234, 231, 233, 227, 229, 229, 231, 231, 233, 233, 228, 229]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369379428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[1]R generation[0..12]: [129, 128, 128, 124, 126, 126, 128, 128, 128, 128, 125, 126]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369390947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369402415Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[2]R flush_numbers[0..12]: [234, 231, 233, 227, 229, 229, 231, 231, 233, 233, 228, 229]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369413654Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[2]R generation[0..12]: [129, 128, 128, 124, 126, 126, 128, 128, 128, 128, 125, 126]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369424732Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369436291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369446289Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369456588Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369466657Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369476645Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"":"downstairs","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369487024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369496303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"0","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369506622Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"1","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.36951705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"client":"2","":"downstairs","session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369527539Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369538567Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"adc152a8-0bc8-4fd3-9fbf-9b0a19e10a3e is now active with session: 4f4e0b05-d661-4aec-b6c1-7f6c336a8643","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369549176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369558475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"session_id":"4f4e0b05-d661-4aec-b6c1-7f6c336a8643"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369568144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Activated sub_volume adc152a8-0bc8-4fd3-9fbf-9b0a19e10a3e","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369576673Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369585321Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.369942635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"task":"crutest"} 01/30 Write block 215 len 40960 data: 7 8 10 14 15 14 13 16 15 14 02/30 Write block 495 len 28672 data: 11 10 9 9 8 8 10 03/30 Write block 239 len 36864 data: 8 6 8 10 10 12 9 10 9 04/30 Write block 487 len 32768 data: 6 8 9 12 13 11 12 11 05/30 Write block 0 len 8192 data: 4 6 06/30 Write block 126 len 40960 data: 10 8 9 10 10 8 8 10 9 8 07/30 Write block 139 len 8192 data: 10 8 08/30 Read block 349 len 24576 09/30 Read block 352 len 12288 10/30 Write block 555 len 28672 data: 8 9 7 9 11 14 14 11/30 Write block 287 len 24576 data: 13 15 13 11 11 9 12/30 Write block 266 len 12288 data: 7 10 10 13/30 Write block 129 len 36864 data: 11 11 9 9 11 10 9 9 9 14/30 Read block 400 len 8192 15/30 Write block 427 len 40960 data: 8 7 7 7 10 10 10 13 11 12 16/30 Read block 304 len 40960 17/30 Write block 47 len 32768 data: 14 14 14 14 15 14 12 11 18/30 Read block 174 len 16384 19/30 Read block 580 len 24576 20/30 Read block 516 len 16384 21/30 Write block 441 len 24576 data: 11 13 11 11 11 12 22/30 Flush 23/30 Write block 262 len 16384 data: 7 8 6 5 24/30 Flush 25/30 Write block 17 len 20480 data: 10 11 12 11 11 26/30 Read block 550 len 4096 27/30 Write block 385 len 12288 data: 13 13 12 28/30 Read block 520 len 36864 29/30 Write block 561 len 12288 data: 15 15 11 30/30 Read block 357 len 4096 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 7 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:45.378961619Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:45.379173751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1724,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 129 235 235 234 T T F <--- 1 020-039 128 128 128 231 231 231 T T F <--- 2 040-059 130 130 128 235 235 233 F F F <--- 3 060-079 124 124 124 227 227 227 F F F 4 080-099 126 126 126 229 229 229 F F F 5 100-119 126 126 126 229 229 229 F F F 6 120-139 130 130 128 235 235 231 F F F <--- 7 140-159 130 130 128 235 235 231 F F F <--- 8 160-179 128 128 128 233 233 233 F F F 9 180-199 128 128 128 233 233 233 F F F 10 200-219 130 130 125 235 235 228 F F T <--- 11 220-239 130 130 126 235 235 229 F F T <--- 12 240-259 130 130 119 235 235 217 F F F <--- 13 260-279 130 130 129 236 236 234 F F F <--- 14 280-299 130 130 123 235 235 223 F F F <--- 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 128 128 128 233 233 233 F F F 19 380-399 128 128 128 233 233 233 T T F <--- 20 400-419 126 126 126 229 229 229 F F F 21 420-439 130 130 126 235 235 229 F F F <--- 22 440-459 130 130 126 235 235 229 F F F <--- 23 460-479 129 129 129 234 234 234 F F F 24 480-499 130 130 126 235 235 229 F F F <--- 25 500-519 130 130 129 235 235 234 F F F <--- 26 520-539 124 124 124 226 226 226 F F F 27 540-559 130 130 124 235 235 227 F F F <--- 28 560-579 130 130 128 235 235 231 T T F <--- 29 580-599 128 128 128 231 231 231 F F F Max gen: 130, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 237 237 237 F F F 1 020-039 131 131 131 237 237 237 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 126 126 126 229 229 229 F F F 5 100-119 126 126 126 229 229 229 F F F 6 120-139 130 130 130 235 235 235 F F F 7 140-159 130 130 130 235 235 235 F F F 8 160-179 128 128 128 233 233 233 F F F 9 180-199 128 128 128 233 233 233 F F F 10 200-219 130 130 130 235 235 235 F F F 11 220-239 130 130 130 235 235 235 F F F 12 240-259 130 130 130 235 235 235 F F F 13 260-279 130 130 130 236 236 236 F F F 14 280-299 130 130 130 235 235 235 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 128 128 128 233 233 233 F F F 19 380-399 131 131 131 237 237 237 F F F 20 400-419 126 126 126 229 229 229 F F F 21 420-439 130 130 130 235 235 235 F F F 22 440-459 130 130 130 235 235 235 F F F 23 460-479 129 129 129 234 234 234 F F F 24 480-499 130 130 130 235 235 235 F F F 25 500-519 130 130 130 235 235 235 F F F 26 520-539 124 124 124 226 226 226 F F F 27 540-559 130 130 130 235 235 235 F F F 28 560-579 131 131 131 237 237 237 F F F 29 580-599 128 128 128 231 231 231 F F F Max gen: 131, Max flush: 237 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.661500962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670164322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670477471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670519146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670529015Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670541283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Crucible b593ff36-0f7c-4608-9973-2e5e4e589b17 has session id: 76a6c576-92fa-4034-9812-5b10ae664d81","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670931703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Upstairs opts: Upstairs UUID: b593ff36-0f7c-4608-9973-2e5e4e589b17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.67095545Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Crucible stats registered with UUID: b593ff36-0f7c-4608-9973-2e5e4e589b17","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.670970988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:59.670980746Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671121368Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671139236Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671149934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671329631Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671352978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"b593ff36-0f7c-4608-9973-2e5e4e589b17 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671382854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671392743Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"0","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671403102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"1","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.67141354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"2","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671423679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671434388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671444856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"ds_connection connected from Ok(127.0.0.1:45449)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.6714955Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"ds_connection connected from Ok(127.0.0.1:38771)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671516307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"ds_connection connected from Ok(127.0.0.1:56759)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.671527426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672273589Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"1","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672294126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"0","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672305985Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"2","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672661199Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[0]R flush_numbers[0..12]: [237, 237, 235, 227, 229, 229, 235, 235, 233, 233, 235, 235]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672686056Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[0]R generation[0..12]: [131, 131, 130, 124, 126, 126, 130, 130, 128, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672698974Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672711262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[1]R flush_numbers[0..12]: [237, 237, 235, 227, 229, 229, 235, 235, 233, 233, 235, 235]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672722761Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[1]R generation[0..12]: [131, 131, 130, 124, 126, 126, 130, 130, 128, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672734069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672745678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[2]R flush_numbers[0..12]: [237, 237, 235, 227, 229, 229, 235, 235, 233, 233, 235, 235]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672756977Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[2]R generation[0..12]: [131, 131, 130, 124, 126, 126, 130, 130, 128, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672768315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672782403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672792632Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672803071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672813179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672823278Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"":"downstairs","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672833907Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672843245Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"0","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672853524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"1","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672863683Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"client":"2","":"downstairs","session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672874111Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.67288388Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"b593ff36-0f7c-4608-9973-2e5e4e589b17 is now active with session: 76a6c576-92fa-4034-9812-5b10ae664d81","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672894679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672904177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"session_id":"76a6c576-92fa-4034-9812-5b10ae664d81"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672913886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Activated sub_volume b593ff36-0f7c-4608-9973-2e5e4e589b17","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672922575Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.672931404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.673298237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"task":"crutest"} 01/30 Flush 02/30 Write block 15 len 16384 data: 9 10 11 12 03/30 Read block 270 len 20480 04/30 Flush 05/30 Write block 43 len 24576 data: 11 13 12 11 15 15 06/30 Read block 177 len 20480 07/30 Flush 08/30 Write block 84 len 12288 data: 8 8 9 09/30 Read block 343 len 28672 10/30 Write block 535 len 28672 data: 11 13 14 15 12 15 16 11/30 Read block 241 len 36864 12/30 Flush 13/30 Write block 159 len 12288 data: 14 13 12 14/30 Read block 527 len 40960 15/30 Write block 395 len 40960 data: 11 11 12 9 6 5 6 9 10 8 16/30 Read block 284 len 24576 17/30 Flush 18/30 Write block 573 len 36864 data: 9 7 10 10 11 9 8 11 8 19/30 Write block 415 len 16384 data: 6 8 11 12 20/30 Flush 21/30 Write block 571 len 12288 data: 7 7 10 22/30 Read block 411 len 36864 23/30 Read block 414 len 24576 24/30 Read block 253 len 12288 25/30 Write block 401 len 20480 data: 7 10 11 9 14 26/30 Read block 514 len 16384 27/30 Read block 444 len 4096 28/30 Read block 53 len 24576 29/30 Read block 339 len 32768 30/30 Write block 234 len 28672 data: 6 11 9 9 10 9 7 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:01:59.683425027Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:01:59.683647658Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1733,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 131 239 239 237 F F F <--- 1 020-039 131 131 131 237 237 237 F F F 2 040-059 132 132 130 240 240 235 F F F <--- 3 060-079 124 124 124 227 227 227 F F F 4 080-099 132 132 126 241 241 229 F F F <--- 5 100-119 126 126 126 229 229 229 F F F 6 120-139 130 130 130 235 235 235 F F F 7 140-159 132 132 130 242 242 235 F F F <--- 8 160-179 132 132 128 242 242 233 F F F <--- 9 180-199 128 128 128 233 233 233 F F F 10 200-219 130 130 130 235 235 235 F F F 11 220-239 130 130 130 235 235 235 T T F <--- 12 240-259 130 130 130 235 235 235 T T F <--- 13 260-279 130 130 130 236 236 236 F F F 14 280-299 130 130 130 235 235 235 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 128 128 128 233 233 233 F F F 19 380-399 132 132 131 242 242 237 F F F <--- 20 400-419 132 132 126 243 243 229 T T F <--- 21 420-439 130 130 130 235 235 235 F F F 22 440-459 130 130 130 235 235 235 F F F 23 460-479 129 129 129 234 234 234 F F F 24 480-499 130 130 130 235 235 235 F F F 25 500-519 130 130 130 235 235 235 F F F 26 520-539 132 132 124 241 241 226 F F F <--- 27 540-559 132 132 130 241 241 235 F F F <--- 28 560-579 132 132 131 243 243 237 T T F <--- 29 580-599 132 132 128 243 243 231 F F F <--- Max gen: 132, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 131 131 131 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 132 132 132 241 241 241 F F F 5 100-119 126 126 126 229 229 229 F F F 6 120-139 130 130 130 235 235 235 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 132 132 132 242 242 242 F F F 9 180-199 128 128 128 233 233 233 F F F 10 200-219 130 130 130 235 235 235 F F F 11 220-239 133 133 133 244 244 244 F F F 12 240-259 133 133 133 244 244 244 F F F 13 260-279 130 130 130 236 236 236 F F F 14 280-299 130 130 130 235 235 235 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 128 128 128 233 233 233 F F F 19 380-399 132 132 132 242 242 242 F F F 20 400-419 133 133 133 244 244 244 F F F 21 420-439 130 130 130 235 235 235 F F F 22 440-459 130 130 130 235 235 235 F F F 23 460-479 129 129 129 234 234 234 F F F 24 480-499 130 130 130 235 235 235 F F F 25 500-519 130 130 130 235 235 235 F F F 26 520-539 132 132 132 241 241 241 F F F 27 540-559 132 132 132 241 241 241 F F F 28 560-579 133 133 133 244 244 244 F F F 29 580-599 132 132 132 243 243 243 F F F Max gen: 133, Max flush: 244 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.960309051Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:13.96751222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.967560603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.974522703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.974791258Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.974822884Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.974831853Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.974843861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Crucible cf5521c8-9d51-4a7f-9766-f0d53b58475e has session id: 58c2818c-9e3d-446b-b968-733b0ffeb5e6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975232451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Upstairs opts: Upstairs UUID: cf5521c8-9d51-4a7f-9766-f0d53b58475e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975283184Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Crucible stats registered with UUID: cf5521c8-9d51-4a7f-9766-f0d53b58475e","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975293753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:13.975303102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975434205Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975465841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975479389Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.9756278Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975652227Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"cf5521c8-9d51-4a7f-9766-f0d53b58475e active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975661765Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975671084Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"0","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975681663Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"1","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975691772Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"2","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.97570152Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975712069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975722358Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35141)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975761603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54400)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.97577827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54785)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.975789049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.9764797Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"2","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976507736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"1","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976657697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"0","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976933831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[0]R flush_numbers[0..12]: [239, 237, 240, 227, 241, 229, 235, 242, 242, 233, 235, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976958958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[0]R generation[0..12]: [132, 131, 132, 124, 132, 126, 130, 132, 132, 128, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976971426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976982905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[1]R flush_numbers[0..12]: [239, 237, 240, 227, 241, 229, 235, 242, 242, 233, 235, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.976993873Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[1]R generation[0..12]: [132, 131, 132, 124, 132, 126, 130, 132, 132, 128, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.97701587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977027249Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[2]R flush_numbers[0..12]: [239, 237, 240, 227, 241, 229, 235, 242, 242, 233, 235, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977038297Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[2]R generation[0..12]: [132, 131, 132, 124, 132, 126, 130, 132, 132, 128, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977049056Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977060405Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977070173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977080252Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977090141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977102069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"":"downstairs","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977112208Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977121227Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"0","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977131125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"1","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977141204Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"client":"2","":"downstairs","session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977151413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977160932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"cf5521c8-9d51-4a7f-9766-f0d53b58475e is now active with session: 58c2818c-9e3d-446b-b968-733b0ffeb5e6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.9771714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977180579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"session_id":"58c2818c-9e3d-446b-b968-733b0ffeb5e6"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977189998Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Activated sub_volume cf5521c8-9d51-4a7f-9766-f0d53b58475e","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977198107Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977206356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.977524665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} 01/30 Write block 374 len 36864 data: 6 4 5 7 8 11 11 14 15 02/30 Read block 152 len 28672 03/30 Read block 425 len 40960 04/30 Read block 31 len 20480 05/30 Write block 250 len 20480 data: 10 11 10 12 8 06/30 Write block 103 len 4096 data: 11 07/30 Write block 388 len 12288 data: 13 14 14 08/30 Write block 503 len 24576 data: 10 11 9 11 11 11 09/30 Read block 383 len 40960 10/30 Write block 183 len 8192 data: 15 13 11/30 Read block 447 len 24576 12/30 Write block 415 len 36864 data: 7 9 12 13 10 9 9 9 8 13/30 Read block 434 len 8192 14/30 Read block 60 len 40960 15/30 Read block 168 len 20480 16/30 Read block 286 len 12288 17/30 Write block 365 len 16384 data: 16 17 16 15 18/30 Write block 469 len 20480 data: 13 14 15 17 15 19/30 Read block 159 len 12288 20/30 Read block 467 len 40960 21/30 Read block 151 len 8192 22/30 Read block 255 len 24576 23/30 Read block 535 len 28672 24/30 Write block 91 len 4096 data: 7 25/30 Flush 26/30 Write block 452 len 36864 data: 8 7 8 5 5 8 10 11 9 27/30 Read block 505 len 32768 28/30 Write block 289 len 20480 data: 14 12 12 10 9 29/30 Read block 564 len 40960 30/30 Write block 436 len 24576 data: 13 13 12 11 12 12 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:13.987047333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:13.987246507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1742,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 131 131 131 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 132 134 134 241 245 245 F F F <--- 5 100-119 126 134 134 229 245 245 F F F <--- 6 120-139 130 130 130 235 235 235 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 132 132 132 242 242 242 F F F 9 180-199 128 134 134 233 245 245 F F F <--- 10 200-219 130 130 130 235 235 235 F F F 11 220-239 133 133 133 244 244 244 F F F 12 240-259 133 134 134 244 245 245 F F F <--- 13 260-279 130 130 130 236 236 236 F F F 14 280-299 130 130 130 235 235 235 F T T <--- 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 128 134 134 233 245 245 T F F <--- 19 380-399 132 134 134 242 245 245 T F F <--- 20 400-419 133 134 134 244 245 245 F F F <--- 21 420-439 130 134 134 235 245 245 F T T <--- 22 440-459 130 130 130 235 235 235 F T T <--- 23 460-479 129 134 134 234 245 245 F T T <--- 24 480-499 130 130 130 235 235 235 F F F 25 500-519 130 134 134 235 245 245 F F F <--- 26 520-539 132 132 132 241 241 241 F F F 27 540-559 132 132 132 241 241 241 F F F 28 560-579 133 133 133 244 244 244 F F F 29 580-599 132 132 132 243 243 243 F F F Max gen: 134, Max flush: 245 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 131 131 131 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 134 134 134 245 245 245 F F F 5 100-119 134 134 134 245 245 245 F F F 6 120-139 130 130 130 235 235 235 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 132 132 132 242 242 242 F F F 9 180-199 134 134 134 245 245 245 F F F 10 200-219 130 130 130 235 235 235 F F F 11 220-239 133 133 133 244 244 244 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 236 236 236 F F F 14 280-299 135 135 135 246 246 246 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 128 128 231 231 231 F F F 18 360-379 134 134 134 245 245 245 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 134 134 134 245 245 245 F F F 21 420-439 135 135 135 246 246 246 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 130 130 130 235 235 235 F F F 25 500-519 134 134 134 245 245 245 F F F 26 520-539 132 132 132 241 241 241 F F F 27 540-559 132 132 132 241 241 241 F F F 28 560-579 133 133 133 244 244 244 F F F 29 580-599 132 132 132 243 243 243 F F F Max gen: 135, Max flush: 246 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.267129694Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:28.275559104Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.275622416Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.283521054Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.283816446Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.283852871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.28386262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.283876148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Crucible f9870994-55e1-47b1-8b92-e7fa67645d74 has session id: 4591f760-145d-4cae-afc9-91dad2f5d8f4","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284333119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Upstairs opts: Upstairs UUID: f9870994-55e1-47b1-8b92-e7fa67645d74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284355046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Crucible stats registered with UUID: f9870994-55e1-47b1-8b92-e7fa67645d74","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284366405Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:28.284376884Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284519915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284539083Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284550641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284733117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284752475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"f9870994-55e1-47b1-8b92-e7fa67645d74 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284782281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284795009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"0","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284806218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"1","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284817527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"2","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284828675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.28486509Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284889477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50554)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284902756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35903)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284914984Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34934)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.284927062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.285626052Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"2","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.285645669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"0","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.285658348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"1","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286052107Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[0]R flush_numbers[0..12]: [239, 237, 240, 227, 245, 245, 235, 242, 242, 245, 235, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286092502Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[0]R generation[0..12]: [132, 131, 132, 124, 134, 134, 130, 132, 132, 134, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.28610541Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286120778Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[1]R flush_numbers[0..12]: [239, 237, 240, 227, 245, 245, 235, 242, 242, 245, 235, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286133136Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[1]R generation[0..12]: [132, 131, 132, 124, 134, 134, 130, 132, 132, 134, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286145175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286157723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[2]R flush_numbers[0..12]: [239, 237, 240, 227, 245, 245, 235, 242, 242, 245, 235, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286169882Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[2]R generation[0..12]: [132, 131, 132, 124, 134, 134, 130, 132, 132, 134, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.28618194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286194568Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286205367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286216616Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286227434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286238273Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"":"downstairs","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286249441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.28625963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"0","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286270879Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"1","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286281967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"client":"2","":"downstairs","session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286295555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286306114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"f9870994-55e1-47b1-8b92-e7fa67645d74 is now active with session: 4591f760-145d-4cae-afc9-91dad2f5d8f4","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286317673Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286327661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"session_id":"4591f760-145d-4cae-afc9-91dad2f5d8f4"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.28633826Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Activated sub_volume f9870994-55e1-47b1-8b92-e7fa67645d74","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286347659Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286357167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.286697413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} 01/30 Write block 143 len 36864 data: 6 8 7 8 7 8 9 8 7 02/30 Read block 573 len 12288 03/30 Read block 174 len 32768 04/30 Write block 574 len 40960 data: 8 11 11 12 10 9 12 9 8 9 05/30 Read block 171 len 20480 06/30 Write block 499 len 36864 data: 9 9 11 10 11 12 10 12 12 07/30 Write block 202 len 40960 data: 9 7 9 8 7 11 8 9 6 4 08/30 Read block 169 len 32768 09/30 Flush 10/30 Write block 345 len 20480 data: 10 11 10 9 12 11/30 Read block 250 len 20480 12/30 Read block 548 len 40960 13/30 Flush 14/30 Write block 273 len 8192 data: 11 12 15/30 Write block 167 len 16384 data: 12 13 11 10 16/30 Write block 115 len 28672 data: 12 9 9 8 8 8 11 17/30 Read block 108 len 36864 18/30 Read block 441 len 36864 19/30 Write block 253 len 4096 data: 13 20/30 Flush 21/30 Write block 476 len 4096 data: 10 22/30 Flush 23/30 Write block 520 len 32768 data: 17 15 15 13 11 11 12 14 24/30 Write block 581 len 32768 data: 10 9 10 10 10 10 9 9 25/30 Read block 513 len 12288 26/30 Write block 274 len 32768 data: 13 14 14 11 10 9 11 9 27/30 Read block 119 len 8192 28/30 Read block 226 len 16384 29/30 Write block 412 len 12288 data: 5 6 7 30/30 Read block 555 len 12288 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:28.297303012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:28.297526303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1751,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 131 131 131 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 134 134 134 245 245 245 F F F 5 100-119 134 136 136 245 249 249 F F F <--- 6 120-139 130 136 136 235 249 249 F F F <--- 7 140-159 132 136 136 242 247 247 F F F <--- 8 160-179 132 136 136 242 249 249 F F F <--- 9 180-199 134 134 134 245 245 245 F F F 10 200-219 130 136 136 235 247 247 F F F <--- 11 220-239 133 133 133 244 244 244 F F F 12 240-259 134 136 136 245 249 249 F F F <--- 13 260-279 130 136 136 236 249 249 F T T <--- 14 280-299 135 135 135 246 246 246 F T T <--- 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 128 136 136 231 248 248 F F F <--- 18 360-379 134 134 134 245 245 245 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 134 134 134 245 245 245 F T T <--- 21 420-439 135 135 135 246 246 246 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 135 136 136 246 250 250 F F F <--- 24 480-499 130 136 136 235 247 247 F F F <--- 25 500-519 134 136 136 245 247 247 F F F <--- 26 520-539 132 132 132 241 241 241 F T T <--- 27 540-559 132 132 132 241 241 241 F F F 28 560-579 133 136 136 244 247 247 F F F <--- 29 580-599 132 136 136 243 247 247 F T T <--- Max gen: 136, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 131 131 131 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 134 134 134 245 245 245 F F F 5 100-119 136 136 136 249 249 249 F F F 6 120-139 136 136 136 249 249 249 F F F 7 140-159 136 136 136 247 247 247 F F F 8 160-179 136 136 136 249 249 249 F F F 9 180-199 134 134 134 245 245 245 F F F 10 200-219 136 136 136 247 247 247 F F F 11 220-239 133 133 133 244 244 244 F F F 12 240-259 136 136 136 249 249 249 F F F 13 260-279 137 137 137 251 251 251 F F F 14 280-299 137 137 137 251 251 251 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 134 134 134 245 245 245 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 135 135 135 246 246 246 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 136 136 136 250 250 250 F F F 24 480-499 136 136 136 247 247 247 F F F 25 500-519 136 136 136 247 247 247 F F F 26 520-539 137 137 137 251 251 251 F F F 27 540-559 132 132 132 241 241 241 F F F 28 560-579 136 136 136 247 247 247 F F F 29 580-599 137 137 137 251 251 251 F F F Max gen: 137, Max flush: 251 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.592505137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.60106806Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601371121Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601402777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601411785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601423624Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Crucible 84be1bec-8150-4070-9ce0-6b8ecd42a311 has session id: a061024b-dbfd-4bc3-a422-4ff48ba51215","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601850799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Upstairs opts: Upstairs UUID: 84be1bec-8150-4070-9ce0-6b8ecd42a311, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601886584Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Crucible stats registered with UUID: 84be1bec-8150-4070-9ce0-6b8ecd42a311","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.601901432Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:42.601926179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.60207589Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602109295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602120854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602272684Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602290172Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"84be1bec-8150-4070-9ce0-6b8ecd42a311 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602320528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602330267Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"0","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602340485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"1","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602350424Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"2","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602360393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602370821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602406557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"ds_connection connected from Ok(127.0.0.1:39500)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602432983Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"ds_connection connected from Ok(127.0.0.1:40130)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602583114Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"ds_connection connected from Ok(127.0.0.1:34355)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.602596202Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603068361Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"2","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603145251Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"0","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603182256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"1","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603515973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[0]R flush_numbers[0..12]: [239, 237, 240, 227, 245, 249, 249, 247, 249, 245, 247, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603554068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[0]R generation[0..12]: [132, 131, 132, 124, 134, 136, 136, 136, 136, 134, 136, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603566067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603577635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[1]R flush_numbers[0..12]: [239, 237, 240, 227, 245, 249, 249, 247, 249, 245, 247, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603588974Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[1]R generation[0..12]: [132, 131, 132, 124, 134, 136, 136, 136, 136, 134, 136, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603600122Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603611581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[2]R flush_numbers[0..12]: [239, 237, 240, 227, 245, 249, 249, 247, 249, 245, 247, 244]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603622789Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[2]R generation[0..12]: [132, 131, 132, 124, 134, 136, 136, 136, 136, 134, 136, 133]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603635898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603647296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603657225Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603667644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603677452Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603687451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"":"downstairs","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.60369803Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603707729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"0","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603717837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"1","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603727896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"client":"2","":"downstairs","session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603738185Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603747673Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"84be1bec-8150-4070-9ce0-6b8ecd42a311 is now active with session: a061024b-dbfd-4bc3-a422-4ff48ba51215","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603758102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603767251Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"session_id":"a061024b-dbfd-4bc3-a422-4ff48ba51215"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.60377676Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Activated sub_volume 84be1bec-8150-4070-9ce0-6b8ecd42a311","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603785108Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.603794607Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.604116036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"task":"crutest"} 01/30 Write block 569 len 16384 data: 10 9 8 8 02/30 Read block 104 len 16384 03/30 Write block 464 len 20480 data: 15 13 10 13 14 04/30 Read block 180 len 16384 05/30 Write block 539 len 36864 data: 13 16 17 17 13 10 9 10 11 06/30 Write block 144 len 4096 data: 9 07/30 Write block 109 len 12288 data: 12 10 8 08/30 Write block 216 len 36864 data: 9 11 15 16 15 14 17 16 15 09/30 Read block 571 len 8192 10/30 Write block 220 len 12288 data: 16 15 18 11/30 Read block 564 len 40960 12/30 Read block 286 len 32768 13/30 Write block 537 len 8192 data: 15 16 14/30 Read block 327 len 12288 15/30 Flush 16/30 Write block 200 len 24576 data: 7 7 10 8 10 9 17/30 Write block 522 len 16384 data: 16 14 12 12 18/30 Read block 283 len 16384 19/30 Read block 282 len 36864 20/30 Read block 567 len 16384 21/30 Write block 25 len 24576 data: 9 10 8 7 8 6 22/30 Read block 386 len 40960 23/30 Read block 14 len 32768 24/30 Read block 338 len 28672 25/30 Flush 26/30 Write block 576 len 32768 data: 12 13 11 10 13 11 10 11 27/30 Write block 173 len 8192 data: 9 8 28/30 Write block 260 len 12288 data: 8 6 8 29/30 Read block 543 len 12288 30/30 Write block 137 len 36864 data: 10 8 11 9 6 8 7 10 8 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 29 5 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:42.61328387Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:42.613502692Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1760,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 138 131 138 253 237 253 F F F <--- 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 134 134 134 245 245 245 F F F 5 100-119 138 136 138 252 249 252 F F F <--- 6 120-139 136 136 136 249 249 249 T F T <--- 7 140-159 138 136 138 252 247 252 T F T <--- 8 160-179 136 136 136 249 249 249 T F T <--- 9 180-199 134 134 134 245 245 245 F F F 10 200-219 138 136 138 253 247 253 F F F <--- 11 220-239 138 133 138 252 244 252 F F F <--- 12 240-259 136 136 136 249 249 249 F F F 13 260-279 137 137 137 251 251 251 T F T <--- 14 280-299 137 137 137 251 251 251 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 134 134 134 245 245 245 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 135 135 135 246 246 246 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 138 136 138 252 250 252 F T F <--- 24 480-499 136 136 136 247 247 247 F F F 25 500-519 136 136 136 247 247 247 F F F 26 520-539 138 137 138 253 251 253 F T F <--- 27 540-559 138 132 138 252 241 252 F T F <--- 28 560-579 138 136 138 252 247 252 T T T <--- 29 580-599 137 137 137 251 251 251 T F T <--- Max gen: 138, Max flush: 253 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 138 138 138 253 253 253 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 124 124 124 227 227 227 F F F 4 080-099 134 134 134 245 245 245 F F F 5 100-119 138 138 138 252 252 252 F F F 6 120-139 139 139 139 254 254 254 F F F 7 140-159 139 139 139 254 254 254 F F F 8 160-179 139 139 139 254 254 254 F F F 9 180-199 134 134 134 245 245 245 F F F 10 200-219 138 138 138 253 253 253 F F F 11 220-239 138 138 138 252 252 252 F F F 12 240-259 136 136 136 249 249 249 F F F 13 260-279 139 139 139 254 254 254 F F F 14 280-299 137 137 137 251 251 251 F F F 15 300-319 128 128 128 231 231 231 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 134 134 134 245 245 245 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 135 135 135 246 246 246 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 136 136 136 247 247 247 F F F 25 500-519 136 136 136 247 247 247 F F F 26 520-539 138 138 138 253 253 253 F F F 27 540-559 138 138 138 252 252 252 F F F 28 560-579 139 139 139 254 254 254 F F F 29 580-599 139 139 139 254 254 254 F F F Max gen: 139, Max flush: 254 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.889841167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.898615802Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.898933151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.898960507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.898969876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.898982264Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Crucible 22941fa5-22f1-4029-a49b-f324ca86d921 has session id: fcedb892-7b78-44bc-890a-76d4d9d01a93","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899413949Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Upstairs opts: Upstairs UUID: 22941fa5-22f1-4029-a49b-f324ca86d921, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899443255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Crucible stats registered with UUID: 22941fa5-22f1-4029-a49b-f324ca86d921","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899453903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:56.899464112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899622682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899655217Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899669106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899824625Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899839374Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"22941fa5-22f1-4029-a49b-f324ca86d921 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899873069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899883038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"0","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899893527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"1","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899903955Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"2","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899914274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899924992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.899935621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"ds_connection connected from Ok(127.0.0.1:64168)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.900008852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"ds_connection connected from Ok(127.0.0.1:63658)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.900042547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"ds_connection connected from Ok(127.0.0.1:59895)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.900132316Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.90071857Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"1","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.900737947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"0","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.900863181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"2","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901145385Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[0]R flush_numbers[0..12]: [239, 253, 240, 227, 245, 252, 254, 254, 254, 245, 253, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901162762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[0]R generation[0..12]: [132, 138, 132, 124, 134, 138, 139, 139, 139, 134, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901174701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901186489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[1]R flush_numbers[0..12]: [239, 253, 240, 227, 245, 252, 254, 254, 254, 245, 253, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901197838Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[1]R generation[0..12]: [132, 138, 132, 124, 134, 138, 139, 139, 139, 134, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901208916Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901220635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[2]R flush_numbers[0..12]: [239, 253, 240, 227, 245, 252, 254, 254, 254, 245, 253, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901231953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[2]R generation[0..12]: [132, 138, 132, 124, 134, 138, 139, 139, 139, 134, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901243212Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.90125481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901265019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901275518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901285756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901297265Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"":"downstairs","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901307714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901317152Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"0","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901327491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"1","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.90133773Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"client":"2","":"downstairs","session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901348038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901357877Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"22941fa5-22f1-4029-a49b-f324ca86d921 is now active with session: fcedb892-7b78-44bc-890a-76d4d9d01a93","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901368476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901377705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"session_id":"fcedb892-7b78-44bc-890a-76d4d9d01a93"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901387543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Activated sub_volume 22941fa5-22f1-4029-a49b-f324ca86d921","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901396182Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901404831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.901763915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"task":"crutest"} 01/30 Write block 198 len 40960 data: 6 7 8 8 11 9 11 10 8 12 02/30 Write block 183 len 12288 data: 16 14 12 03/30 Flush 04/30 Write block 40 len 4096 data: 11 05/30 Read block 476 len 20480 06/30 Read block 559 len 4096 07/30 Write block 360 len 28672 data: 16 14 19 18 20 17 18 08/30 Write block 272 len 4096 data: 10 09/30 Write block 449 len 4096 data: 10 10/30 Write block 148 len 24576 data: 9 10 9 8 9 8 11/30 Read block 86 len 12288 12/30 Read block 589 len 20480 13/30 Write block 71 len 36864 data: 7 6 5 6 7 6 5 6 7 14/30 Write block 312 len 24576 data: 13 12 12 11 10 11 15/30 Read block 408 len 12288 16/30 Read block 471 len 40960 17/30 Write block 272 len 16384 data: 11 12 14 15 18/30 Write block 545 len 8192 data: 10 11 19/30 Write block 299 len 24576 data: 6 8 10 9 9 10 20/30 Read block 54 len 16384 21/30 Write block 437 len 28672 data: 14 13 12 13 13 14 12 22/30 Write block 28 len 28672 data: 8 9 7 6 8 8 8 23/30 Flush 24/30 Write block 159 len 20480 data: 15 14 13 13 11 25/30 Write block 495 len 12288 data: 12 11 10 26/30 Read block 568 len 32768 27/30 Write block 409 len 4096 data: 12 28/30 Read block 216 len 36864 29/30 Read block 341 len 16384 30/30 Write block 539 len 28672 data: 14 17 18 18 14 11 11 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Write 3 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:02:56.910338746Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:02:56.910565466Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1769,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 140 140 138 256 256 253 F F F <--- 2 040-059 140 140 132 256 256 240 F F F <--- 3 060-079 140 140 124 256 256 227 F F F <--- 4 080-099 134 134 134 245 245 245 F F F 5 100-119 138 138 138 252 252 252 F F F 6 120-139 139 139 139 254 254 254 F F F 7 140-159 140 140 139 256 256 254 T T F <--- 8 160-179 139 139 139 254 254 254 T T F <--- 9 180-199 140 140 134 255 255 245 F F F <--- 10 200-219 140 140 138 255 255 253 F F F <--- 11 220-239 138 138 138 252 252 252 F F F 12 240-259 136 136 136 249 249 249 F F F 13 260-279 140 140 139 256 256 254 F F F <--- 14 280-299 140 140 137 256 256 251 F F F <--- 15 300-319 140 140 128 256 256 231 F F F <--- 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 140 140 134 256 256 245 F F F <--- 19 380-399 134 134 134 245 245 245 F F F 20 400-419 137 137 137 251 251 251 T T F <--- 21 420-439 140 140 135 256 256 246 F F F <--- 22 440-459 140 140 135 256 256 246 F F F <--- 23 460-479 138 138 138 252 252 252 F F F 24 480-499 136 136 136 247 247 247 T T F <--- 25 500-519 136 136 136 247 247 247 F F F 26 520-539 138 138 138 253 253 253 T T F <--- 27 540-559 140 140 138 256 256 252 T T F <--- 28 560-579 139 139 139 254 254 254 F F F 29 580-599 139 139 139 254 254 254 F F F Max gen: 140, Max flush: 256 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 140 140 140 256 256 256 F F F 2 040-059 140 140 140 256 256 256 F F F 3 060-079 140 140 140 256 256 256 F F F 4 080-099 134 134 134 245 245 245 F F F 5 100-119 138 138 138 252 252 252 F F F 6 120-139 139 139 139 254 254 254 F F F 7 140-159 141 141 141 257 257 257 F F F 8 160-179 141 141 141 257 257 257 F F F 9 180-199 140 140 140 255 255 255 F F F 10 200-219 140 140 140 255 255 255 F F F 11 220-239 138 138 138 252 252 252 F F F 12 240-259 136 136 136 249 249 249 F F F 13 260-279 140 140 140 256 256 256 F F F 14 280-299 140 140 140 256 256 256 F F F 15 300-319 140 140 140 256 256 256 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 140 140 140 256 256 256 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 141 141 141 257 257 257 F F F 21 420-439 140 140 140 256 256 256 F F F 22 440-459 140 140 140 256 256 256 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 141 141 141 257 257 257 F F F 25 500-519 136 136 136 247 247 247 F F F 26 520-539 141 141 141 257 257 257 F F F 27 540-559 141 141 141 257 257 257 F F F 28 560-579 139 139 139 254 254 254 F F F 29 580-599 139 139 139 254 254 254 F F F Max gen: 141, Max flush: 257 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.203973154Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.212806212Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213112142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213143658Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213156286Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213169075Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Crucible 59f5852c-b261-4eca-8e8f-7358da7133c9 has session id: 3625167f-f936-4411-92d9-385e48976051","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213602059Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Upstairs opts: Upstairs UUID: 59f5852c-b261-4eca-8e8f-7358da7133c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213638444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Crucible stats registered with UUID: 59f5852c-b261-4eca-8e8f-7358da7133c9","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213650322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:11.213660441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213805812Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.2138244Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.213835828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214012086Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214026914Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"59f5852c-b261-4eca-8e8f-7358da7133c9 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214065299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214075667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"0","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214086346Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"1","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214098854Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"2","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214109513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214120432Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.21413119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"ds_connection connected from Ok(127.0.0.1:57121)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214195442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"ds_connection connected from Ok(127.0.0.1:37071)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214223038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"ds_connection connected from Ok(127.0.0.1:55719)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214235327Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214893852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"1","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.214949074Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"0","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215003058Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"2","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215309628Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[0]R flush_numbers[0..12]: [239, 256, 256, 256, 245, 252, 254, 257, 257, 255, 255, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215346353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[0]R generation[0..12]: [132, 140, 140, 140, 134, 138, 139, 141, 141, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215358402Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.21537023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[1]R flush_numbers[0..12]: [239, 256, 256, 256, 245, 252, 254, 257, 257, 255, 255, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215384248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[1]R generation[0..12]: [132, 140, 140, 140, 134, 138, 139, 141, 141, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215395827Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215407545Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[2]R flush_numbers[0..12]: [239, 256, 256, 256, 245, 252, 254, 257, 257, 255, 255, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215419044Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[2]R generation[0..12]: [132, 140, 140, 140, 134, 138, 139, 141, 141, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215430342Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215442081Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215452489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215463158Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215473197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215483435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"":"downstairs","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215493934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215503363Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"0","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215513791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"1","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.21552422Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"client":"2","":"downstairs","session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215534769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215545947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"59f5852c-b261-4eca-8e8f-7358da7133c9 is now active with session: 3625167f-f936-4411-92d9-385e48976051","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215556736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-05-14T21:03:11.215566245Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"session_id":"3625167f-f936-4411-92d9-385e48976051"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215581623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Activated sub_volume 59f5852c-b261-4eca-8e8f-7358da7133c9","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215590252Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.21559908Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.215857367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"task":"crutest"} 01/30 Write block 117 len 40960 data: 10 9 9 9 12 11 11 13 13 11 02/30 Read block 262 len 24576 03/30 Flush 04/30 Write block 172 len 28672 data: 9 10 9 10 10 13 12 05/30 Write block 67 len 28672 data: 11 10 8 7 8 7 6 06/30 Read block 496 len 40960 07/30 Read block 423 len 40960 08/30 Write block 155 len 32768 data: 9 11 13 14 16 15 14 14 09/30 Read block 467 len 32768 10/30 Write block 24 len 16384 data: 10 10 11 9 11/30 Write block 87 len 8192 data: 10 10 12/30 Read block 201 len 16384 13/30 Read block 279 len 32768 14/30 Write block 579 len 8192 data: 11 14 15/30 Write block 60 len 40960 data: 8 12 13 12 10 10 11 12 11 9 16/30 Write block 522 len 32768 data: 17 15 13 13 13 15 14 11 17/30 Read block 287 len 8192 18/30 Read block 381 len 28672 19/30 Write block 78 len 20480 data: 7 8 8 7 9 20/30 Write block 570 len 20480 data: 10 9 9 11 9 21/30 Write block 152 len 28672 data: 10 9 11 10 12 14 15 22/30 Write block 550 len 8192 data: 9 8 23/30 Flush 24/30 Write block 486 len 4096 data: 7 25/30 Write block 50 len 4096 data: 15 26/30 Read block 386 len 8192 27/30 Read block 266 len 4096 28/30 Read block 421 len 40960 29/30 Write block 300 len 32768 data: 9 11 10 10 11 10 14 14 30/30 Write block 196 len 20480 data: 6 6 7 8 9 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 7 Done Done Done false 1004 Acked Write 7 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 26 30 Done 25 25 1 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:11.224448066Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:11.224681266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1778,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 142 142 140 259 259 256 F F F <--- 2 040-059 140 140 140 256 256 256 T T F <--- 3 060-079 142 142 140 259 259 256 F F F <--- 4 080-099 142 142 134 259 259 245 F F F <--- 5 100-119 142 142 142 258 258 258 F F F 6 120-139 142 142 142 258 258 258 F F F 7 140-159 142 142 141 259 259 257 F F F <--- 8 160-179 142 142 141 259 259 257 F F T <--- 9 180-199 140 140 140 255 255 255 T T F <--- 10 200-219 140 140 140 255 255 255 T T F <--- 11 220-239 138 138 138 252 252 252 F F F 12 240-259 136 136 136 249 249 249 F F F 13 260-279 140 140 140 256 256 256 F F F 14 280-299 140 140 140 256 256 256 F F F 15 300-319 140 140 140 256 256 256 T T F <--- 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 140 140 140 256 256 256 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 141 141 141 257 257 257 F F F 21 420-439 140 140 140 256 256 256 F F F 22 440-459 140 140 140 256 256 256 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 141 141 141 257 257 257 T T F <--- 25 500-519 136 136 136 247 247 247 F F F 26 520-539 142 142 141 259 259 257 F F F <--- 27 540-559 142 142 141 259 259 257 F F F <--- 28 560-579 142 142 139 259 259 254 F F F <--- 29 580-599 142 142 139 259 259 254 F F F <--- Max gen: 142, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 142 142 142 259 259 259 F F F 2 040-059 143 143 143 260 260 260 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 142 142 142 258 258 258 F F F 7 140-159 142 142 142 259 259 259 F F F 8 160-179 142 142 142 259 259 259 F F F 9 180-199 143 143 143 260 260 260 F F F 10 200-219 143 143 143 260 260 260 F F F 11 220-239 138 138 138 252 252 252 F F F 12 240-259 136 136 136 249 249 249 F F F 13 260-279 140 140 140 256 256 256 F F F 14 280-299 140 140 140 256 256 256 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F F F 18 360-379 140 140 140 256 256 256 F F F 19 380-399 134 134 134 245 245 245 F F F 20 400-419 141 141 141 257 257 257 F F F 21 420-439 140 140 140 256 256 256 F F F 22 440-459 140 140 140 256 256 256 F F F 23 460-479 138 138 138 252 252 252 F F F 24 480-499 143 143 143 260 260 260 F F F 25 500-519 136 136 136 247 247 247 F F F 26 520-539 142 142 142 259 259 259 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 142 142 142 259 259 259 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 143, Max flush: 260 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.519495072Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:25.527408168Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.527458632Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535118291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535402484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.53543428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535443879Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535456447Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Crucible 4cb2e51e-a129-496b-8c5d-8c1a2b58c47c has session id: a39dac16-9a15-4fe8-9ac2-aaaf40864dbb","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535830469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Upstairs opts: Upstairs UUID: 4cb2e51e-a129-496b-8c5d-8c1a2b58c47c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535859705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Crucible stats registered with UUID: 4cb2e51e-a129-496b-8c5d-8c1a2b58c47c","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.535870754Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:25.535909079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536062149Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536094585Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536106613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536272442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.53628627Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"4cb2e51e-a129-496b-8c5d-8c1a2b58c47c active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536324335Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536334744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"0","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536345252Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"1","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536355931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"2","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.5363662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536400175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536428712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43325)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.53644221Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43442)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536453478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52902)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.536464427Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537160907Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"2","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537202371Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"1","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.5372162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"0","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537543807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[0]R flush_numbers[0..12]: [239, 259, 260, 259, 259, 258, 258, 259, 259, 260, 260, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537571404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[0]R generation[0..12]: [132, 142, 143, 142, 142, 142, 142, 142, 142, 143, 143, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537583692Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537595661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[1]R flush_numbers[0..12]: [239, 259, 260, 259, 259, 258, 258, 259, 259, 260, 260, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537607299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[1]R generation[0..12]: [132, 142, 143, 142, 142, 142, 142, 142, 142, 143, 143, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537618748Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537630686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[2]R flush_numbers[0..12]: [239, 259, 260, 259, 259, 258, 258, 259, 259, 260, 260, 252]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537642255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[2]R generation[0..12]: [132, 142, 143, 142, 142, 142, 142, 142, 142, 143, 143, 138]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537655503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537667351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.53767775Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537688409Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537698647Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537708766Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"":"downstairs","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537719265Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537728853Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"0","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537739262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"1","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537749861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"client":"2","":"downstairs","session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537760369Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537770348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"4cb2e51e-a129-496b-8c5d-8c1a2b58c47c is now active with session: a39dac16-9a15-4fe8-9ac2-aaaf40864dbb","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537781077Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537790785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"session_id":"a39dac16-9a15-4fe8-9ac2-aaaf40864dbb"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537800794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Activated sub_volume 4cb2e51e-a129-496b-8c5d-8c1a2b58c47c","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.537809723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-05-14T21:03:25.537819992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.538116923Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} 01/30 Write block 527 len 40960 data: 16 15 12 11 11 9 10 9 12 14 02/30 Write block 236 len 32768 data: 10 10 11 10 8 9 11 11 03/30 Read block 450 len 28672 04/30 Write block 396 len 20480 data: 12 13 10 7 6 05/30 Write block 457 len 32768 data: 9 11 12 10 11 11 12 16 06/30 Write block 28 len 4096 data: 9 07/30 Write block 144 len 32768 data: 11 9 9 8 10 11 10 9 08/30 Write block 424 len 8192 data: 6 6 09/30 Read block 312 len 8192 10/30 Write block 216 len 20480 data: 10 12 16 17 17 11/30 Read block 278 len 8192 12/30 Read block 514 len 36864 13/30 Write block 527 len 24576 data: 17 16 13 12 12 10 14/30 Write block 363 len 36864 data: 19 21 18 19 17 16 15 13 9 15/30 Write block 578 len 4096 data: 12 16/30 Read block 72 len 20480 17/30 Read block 52 len 16384 18/30 Write block 504 len 4096 data: 13 19/30 Read block 115 len 40960 20/30 Read block 59 len 40960 21/30 Write block 401 len 4096 data: 8 22/30 Write block 355 len 24576 data: 12 9 9 13 14 17 23/30 Write block 41 len 32768 data: 11 11 12 14 13 12 16 16 24/30 Read block 549 len 32768 25/30 Write block 132 len 20480 data: 10 12 11 10 10 26/30 Read block 515 len 16384 27/30 Read block 116 len 16384 28/30 Read block 26 len 32768 29/30 Read block 76 len 28672 30/30 Write block 287 len 12288 data: 14 16 15 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 7 29 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:25.546534814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:25.546761255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1787,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 142 142 142 259 259 259 T T T 2 040-059 143 143 143 260 260 260 F T T <--- 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 142 142 142 258 258 258 F T T <--- 7 140-159 142 142 142 259 259 259 T T T 8 160-179 142 142 142 259 259 259 F F F 9 180-199 143 143 143 260 260 260 F F F 10 200-219 143 143 143 260 260 260 F T T <--- 11 220-239 138 138 138 252 252 252 T T T 12 240-259 136 136 136 249 249 249 T T T 13 260-279 140 140 140 256 256 256 F F F 14 280-299 140 140 140 256 256 256 F T T <--- 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 136 136 136 248 248 248 F T T <--- 18 360-379 140 140 140 256 256 256 F T T <--- 19 380-399 134 134 134 245 245 245 T T T 20 400-419 141 141 141 257 257 257 T T T 21 420-439 140 140 140 256 256 256 F T T <--- 22 440-459 140 140 140 256 256 256 T T T 23 460-479 138 138 138 252 252 252 T T T 24 480-499 143 143 143 260 260 260 F F F 25 500-519 136 136 136 247 247 247 F T T <--- 26 520-539 142 142 142 259 259 259 T T T 27 540-559 142 142 142 259 259 259 F F F 28 560-579 142 142 142 259 259 259 F T T <--- 29 580-599 142 142 142 259 259 259 F F F Max gen: 143, Max flush: 260 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 223c223 < 16, --- > 17, 404c404 < 7, --- > 8, 529a530 > 17, 531c532 < 15, --- > 13, 533,535c534 < 11, < 11, < 9, --- > 12, 536a536 > 10, diff found after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 144 144 144 261 261 261 F F F 2 040-059 144 144 144 261 261 261 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 144 144 144 261 261 261 F F F 7 140-159 144 144 144 261 261 261 F F F 8 160-179 142 142 142 259 259 259 F F F 9 180-199 143 143 143 260 260 260 F F F 10 200-219 144 144 144 261 261 261 F F F 11 220-239 144 144 144 261 261 261 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 140 140 140 256 256 256 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 144 144 144 261 261 261 F F F 18 360-379 144 144 144 261 261 261 F F F 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 144 144 144 261 261 261 F F F 23 460-479 144 144 144 261 261 261 F F F 24 480-499 143 143 143 260 260 260 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 144 144 144 261 261 261 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 144 144 144 261 261 261 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 144, Max flush: 261 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.858504371Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867157492Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867465622Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867496328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867505667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867518065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Crucible 3767d8dc-1100-4308-9829-85b32c8c9ba3 has session id: b8c56758-90d6-43ca-98e4-4dff5e2daae6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867899766Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Upstairs opts: Upstairs UUID: 3767d8dc-1100-4308-9829-85b32c8c9ba3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867919773Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Crucible stats registered with UUID: 3767d8dc-1100-4308-9829-85b32c8c9ba3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.867930612Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:39.86794047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868086682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868118307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868130426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868285386Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868304903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"3767d8dc-1100-4308-9829-85b32c8c9ba3 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868335399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868345398Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"0","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868355877Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"1","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868366405Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"2","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868376834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868387483Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868398131Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61876)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.86848547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53435)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868566859Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46541)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.868580898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869206727Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"0","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869262699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"1","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869316662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"2","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869621013Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[0]R flush_numbers[0..12]: [239, 261, 261, 259, 259, 258, 261, 261, 259, 260, 261, 261]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869656688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[0]R generation[0..12]: [132, 144, 144, 142, 142, 142, 144, 144, 142, 143, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869668847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869680915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[1]R flush_numbers[0..12]: [239, 261, 261, 259, 259, 258, 261, 261, 259, 260, 261, 261]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869692354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[1]R generation[0..12]: [132, 144, 144, 142, 142, 142, 144, 144, 142, 143, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869703692Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869715431Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[2]R flush_numbers[0..12]: [239, 261, 261, 259, 259, 258, 261, 261, 259, 260, 261, 261]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869726769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[2]R generation[0..12]: [132, 144, 144, 142, 142, 142, 144, 144, 142, 143, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869738158Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869749856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Max found gen is 145","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869760025Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Generation requested: 146 >= found:145","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869770434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869781932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869792071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"":"downstairs","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.86980274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869812268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"0","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869823167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"1","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869834066Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"client":"2","":"downstairs","session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869845064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869855193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"3767d8dc-1100-4308-9829-85b32c8c9ba3 is now active with session: b8c56758-90d6-43ca-98e4-4dff5e2daae6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869865951Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.86987552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"session_id":"b8c56758-90d6-43ca-98e4-4dff5e2daae6"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869885559Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Activated sub_volume 3767d8dc-1100-4308-9829-85b32c8c9ba3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869894258Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.869903207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.870214986Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"task":"crutest"} 01/30 Write block 525 len 24576 data: 14 14 17 16 13 12 02/30 Read block 379 len 24576 03/30 Write block 47 len 4096 data: 17 04/30 Read block 494 len 20480 05/30 Write block 155 len 8192 data: 11 13 06/30 Write block 353 len 40960 data: 12 13 13 10 10 14 15 18 15 20 07/30 Write block 37 len 24576 data: 8 9 9 12 12 12 08/30 Read block 171 len 12288 09/30 Read block 44 len 40960 10/30 Read block 28 len 4096 11/30 Write block 11 len 40960 data: 10 7 7 7 10 11 12 13 13 12 12/30 Write block 355 len 8192 data: 14 11 13/30 Write block 141 len 40960 data: 7 9 8 12 10 10 9 11 12 11 14/30 Read block 563 len 16384 15/30 Write block 227 len 36864 data: 14 11 7 6 6 6 8 7 12 16/30 Read block 448 len 28672 17/30 Write block 359 len 36864 data: 16 19 16 21 20 22 19 20 18 18/30 Read block 222 len 36864 19/30 Read block 147 len 4096 20/30 Write block 354 len 28672 data: 14 15 12 11 15 17 20 21/30 Read block 512 len 12288 22/30 Read block 491 len 28672 23/30 Read block 56 len 28672 24/30 Write block 185 len 12288 data: 13 15 11 25/30 Read block 315 len 16384 26/30 Write block 171 len 36864 data: 13 10 11 10 11 11 14 13 9 27/30 Read block 303 len 32768 28/30 Read block 573 len 12288 29/30 Read block 318 len 28672 30/30 Read block 168 len 32768 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:39.880058883Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:39.880268726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1799,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 T T F <--- 1 020-039 144 144 144 261 261 261 T T F <--- 2 040-059 144 144 144 261 261 261 T T F <--- 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 144 144 144 261 261 261 F F F 7 140-159 144 144 144 261 261 261 T T F <--- 8 160-179 142 142 142 259 259 259 T T F <--- 9 180-199 143 143 143 260 260 260 T T F <--- 10 200-219 144 144 144 261 261 261 F F F 11 220-239 144 144 144 261 261 261 T T F <--- 12 240-259 144 144 144 261 261 261 F F F 13 260-279 140 140 140 256 256 256 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 144 144 144 261 261 261 T T F <--- 18 360-379 144 144 144 261 261 261 T T F <--- 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 144 144 144 261 261 261 F F F 23 460-479 144 144 144 261 261 261 F F F 24 480-499 143 143 143 260 260 260 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 144 144 144 261 261 261 T T T 27 540-559 142 142 142 259 259 259 F F F 28 560-579 144 144 144 261 261 261 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 144, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 262 262 262 F F F 1 020-039 145 145 145 262 262 262 F F F 2 040-059 145 145 145 262 262 262 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 144 144 144 261 261 261 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 144 144 144 261 261 261 F F F 11 220-239 145 145 145 262 262 262 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 140 140 140 256 256 256 F F F 14 280-299 144 144 144 261 261 261 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 145 145 145 262 262 262 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 144 144 144 261 261 261 F F F 23 460-479 144 144 144 261 261 261 F F F 24 480-499 143 143 143 260 260 260 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 145 145 145 262 262 262 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 144 144 144 261 261 261 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 145, Max flush: 262 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.159448434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.167817062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168128701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168159977Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168169106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168181175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Crucible af4dcecd-d1f2-45e3-9a4e-244c28034fda has session id: 1b704f04-434d-42e6-bc82-9a2bab0bbf92","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168549597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Upstairs opts: Upstairs UUID: af4dcecd-d1f2-45e3-9a4e-244c28034fda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168570544Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Crucible stats registered with UUID: af4dcecd-d1f2-45e3-9a4e-244c28034fda","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168580813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:54.168590292Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168729414Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.16876067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168772178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168906901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168920769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"af4dcecd-d1f2-45e3-9a4e-244c28034fda active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168954385Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168963993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"0","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168976202Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"1","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.1689861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"2","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.168996409Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169006618Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169016916Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"ds_connection connected from Ok(127.0.0.1:37636)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169071229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"ds_connection connected from Ok(127.0.0.1:60529)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169099976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"ds_connection connected from Ok(127.0.0.1:60939)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169111354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169737263Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"0","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169756781Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"1","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.169799875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"2","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170105776Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[0]R flush_numbers[0..12]: [262, 262, 262, 259, 259, 258, 261, 262, 262, 262, 261, 262]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170140201Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[0]R generation[0..12]: [145, 145, 145, 142, 142, 142, 144, 145, 145, 145, 144, 145]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.17015185Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170165858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[1]R flush_numbers[0..12]: [262, 262, 262, 259, 259, 258, 261, 262, 262, 262, 261, 262]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170177126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[1]R generation[0..12]: [145, 145, 145, 142, 142, 142, 144, 145, 145, 145, 144, 145]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170188035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170199374Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[2]R flush_numbers[0..12]: [262, 262, 262, 259, 259, 258, 261, 262, 262, 262, 261, 262]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170210512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[2]R generation[0..12]: [145, 145, 145, 142, 142, 142, 144, 145, 145, 145, 144, 145]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170221401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170232659Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170242208Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Generation requested: 148 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170252387Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170262115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170271934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"":"downstairs","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170282233Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170291142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"0","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.17030133Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"1","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170311289Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"client":"2","":"downstairs","session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170322758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170332096Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"af4dcecd-d1f2-45e3-9a4e-244c28034fda is now active with session: 1b704f04-434d-42e6-bc82-9a2bab0bbf92","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170342275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170351364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"session_id":"1b704f04-434d-42e6-bc82-9a2bab0bbf92"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170360713Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Activated sub_volume af4dcecd-d1f2-45e3-9a4e-244c28034fda","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170369002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.17037747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.170670633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"task":"crutest"} 01/30 Write block 152 len 4096 data: 11 02/30 Write block 224 len 24576 data: 16 16 14 15 12 8 03/30 Read block 70 len 36864 04/30 Flush 05/30 Write block 242 len 28672 data: 12 12 13 10 11 10 9 06/30 Read block 318 len 36864 07/30 Write block 6 len 16384 data: 8 9 9 9 08/30 Flush 09/30 Write block 465 len 24576 data: 14 11 14 15 14 15 10/30 Flush 11/30 Write block 200 len 20480 data: 10 9 12 10 12 12/30 Write block 481 len 12288 data: 8 9 9 13/30 Read block 369 len 28672 14/30 Write block 52 len 16384 data: 15 13 12 12 15/30 Read block 76 len 8192 16/30 Write block 489 len 20480 data: 10 13 14 12 13 17/30 Read block 541 len 4096 18/30 Write block 271 len 24576 data: 12 12 13 15 16 15 19/30 Flush 20/30 Write block 570 len 40960 data: 11 10 10 12 10 12 13 14 13 12 21/30 Read block 50 len 40960 22/30 Write block 2 len 40960 data: 6 4 6 7 9 10 10 10 8 11 23/30 Read block 141 len 4096 24/30 Flush 25/30 Write block 21 len 16384 data: 12 8 10 11 26/30 Read block 76 len 24576 27/30 Read block 231 len 24576 28/30 Write block 285 len 4096 data: 11 29/30 Write block 215 len 32768 data: 8 11 13 17 18 17 16 19 30/30 Read block 417 len 24576 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 5 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 15 16 Done 19 20 5 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:03:54.179750568Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:03:54.179947373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1808,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 267 267 264 F F F <--- 1 020-039 145 145 145 262 262 262 T T F <--- 2 040-059 148 148 145 266 266 262 F F T <--- 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 144 144 144 261 261 261 F F F 7 140-159 148 148 148 263 263 263 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 148 148 144 266 266 261 T T T <--- 11 220-239 148 148 148 263 263 263 T T F <--- 12 240-259 148 148 148 264 264 264 F F F 13 260-279 148 148 140 266 266 256 F F F <--- 14 280-299 144 144 144 261 261 261 T T F <--- 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 145 145 145 262 262 262 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 144 144 144 261 261 261 F F F 23 460-479 148 148 148 265 265 265 F F F 24 480-499 148 148 143 266 266 260 F F T <--- 25 500-519 144 144 144 261 261 261 F F F 26 520-539 145 145 145 262 262 262 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 148 148 144 267 267 261 F F F <--- 29 580-599 142 142 142 259 259 259 F F F Max gen: 148, Max flush: 267 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 267 267 267 F F F 1 020-039 149 149 149 268 268 268 F F F 2 040-059 148 148 148 266 266 266 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 F F F 6 120-139 144 144 144 261 261 261 F F F 7 140-159 148 148 148 263 263 263 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 149 149 149 268 268 268 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 148 148 148 264 264 264 F F F 13 260-279 148 148 148 266 266 266 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 F F F 17 340-359 145 145 145 262 262 262 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 144 144 144 261 261 261 F F F 23 460-479 148 148 148 265 265 265 F F F 24 480-499 148 148 148 266 266 266 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 145 145 145 262 262 262 F F F 27 540-559 142 142 142 259 259 259 F F F 28 560-579 148 148 148 267 267 267 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 149, Max flush: 268 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.448180177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.456858554Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457164705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457196481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457205699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457217778Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Crucible 9570e334-b67d-4b95-9ab5-3ae1982fad1f has session id: 651a8726-f9d4-4e59-b33b-6c49ce22f928","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457599259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Upstairs opts: Upstairs UUID: 9570e334-b67d-4b95-9ab5-3ae1982fad1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457616426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Crucible stats registered with UUID: 9570e334-b67d-4b95-9ab5-3ae1982fad1f","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457626815Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:08.457638403Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45782276Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457839997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.457850536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458036682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45805221Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"9570e334-b67d-4b95-9ab5-3ae1982fad1f active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458092745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458102563Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"0","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458112712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"1","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458122811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"2","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45813277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458143218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458153597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"ds_connection connected from Ok(127.0.0.1:65048)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45820751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"ds_connection connected from Ok(127.0.0.1:39468)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458219048Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"ds_connection connected from Ok(127.0.0.1:51058)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458229747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458942645Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"1","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.458970411Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"0","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459022424Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"2","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459340483Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[0]R flush_numbers[0..12]: [267, 268, 266, 259, 259, 258, 261, 263, 262, 262, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459371659Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[0]R generation[0..12]: [148, 149, 148, 142, 142, 142, 144, 148, 145, 145, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459383698Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459395446Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[1]R flush_numbers[0..12]: [267, 268, 266, 259, 259, 258, 261, 263, 262, 262, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459406815Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[1]R generation[0..12]: [148, 149, 148, 142, 142, 142, 144, 148, 145, 145, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459418053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459429682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[2]R flush_numbers[0..12]: [267, 268, 266, 259, 259, 258, 261, 263, 262, 262, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45944087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[2]R generation[0..12]: [148, 149, 148, 142, 142, 142, 144, 148, 145, 145, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459452349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459464107Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459477046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459487674Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459497643Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459507532Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"":"downstairs","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45951764Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459527079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"0","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459537228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"1","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459547357Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"client":"2","":"downstairs","session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459557845Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459567474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"9570e334-b67d-4b95-9ab5-3ae1982fad1f is now active with session: 651a8726-f9d4-4e59-b33b-6c49ce22f928","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459577953Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459587211Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"session_id":"651a8726-f9d4-4e59-b33b-6c49ce22f928"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.45959705Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Activated sub_volume 9570e334-b67d-4b95-9ab5-3ae1982fad1f","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459605549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459614218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.459966462Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"task":"crutest"} 01/30 Write block 137 len 20480 data: 11 9 12 10 8 02/30 Read block 553 len 24576 03/30 Read block 421 len 12288 04/30 Read block 433 len 32768 05/30 Read block 562 len 28672 06/30 Write block 246 len 20480 data: 12 11 10 9 11 07/30 Write block 442 len 12288 data: 15 13 12 08/30 Write block 138 len 4096 data: 10 09/30 Read block 162 len 28672 10/30 Read block 334 len 40960 11/30 Write block 9 len 16384 data: 11 9 12 8 12/30 Read block 486 len 32768 13/30 Read block 448 len 36864 14/30 Write block 196 len 4096 data: 7 15/30 Read block 77 len 24576 16/30 Write block 558 len 8192 data: 10 12 17/30 Write block 333 len 12288 data: 12 10 9 18/30 Write block 142 len 16384 data: 10 9 13 11 19/30 Write block 345 len 16384 data: 11 12 11 10 20/30 Read block 282 len 32768 21/30 Read block 63 len 16384 22/30 Read block 238 len 12288 23/30 Write block 350 len 32768 data: 14 13 12 13 15 16 13 12 24/30 Read block 363 len 12288 25/30 Write block 109 len 36864 data: 13 11 9 8 11 11 13 10 11 26/30 Read block 385 len 8192 27/30 Write block 492 len 16384 data: 13 14 12 13 28/30 Read block 91 len 32768 29/30 Write block 247 len 16384 data: 12 11 10 12 30/30 Read block 238 len 40960 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:08.468846334Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:08.469065425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1817,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 267 267 267 T T F <--- 1 020-039 149 149 149 268 268 268 F F F 2 040-059 148 148 148 266 266 266 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 142 142 142 258 258 258 T T F <--- 6 120-139 144 144 144 261 261 261 T T F <--- 7 140-159 148 148 148 263 263 263 T T F <--- 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 T T F <--- 10 200-219 149 149 149 268 268 268 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 148 148 148 264 264 264 T T F <--- 13 260-279 148 148 148 266 266 266 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 124 124 124 225 225 225 T T F <--- 17 340-359 145 145 145 262 262 262 T T F <--- 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 144 144 144 261 261 261 T T F <--- 23 460-479 148 148 148 265 265 265 F F F 24 480-499 148 148 148 266 266 266 T T F <--- 25 500-519 144 144 144 261 261 261 F F F 26 520-539 145 145 145 262 262 262 F F F 27 540-559 142 142 142 259 259 259 T T F <--- 28 560-579 148 148 148 267 267 267 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 149, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 269 269 269 F F F 1 020-039 149 149 149 268 268 268 F F F 2 040-059 148 148 148 266 266 266 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 142 142 142 259 259 259 F F F 5 100-119 150 150 150 269 269 269 F F F 6 120-139 150 150 150 269 269 269 F F F 7 140-159 150 150 150 269 269 269 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 150 150 150 269 269 269 F F F 10 200-219 149 149 149 268 268 268 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 150 150 150 269 269 269 F F F 13 260-279 148 148 148 266 266 266 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 143 143 143 260 260 260 F F F 16 320-339 150 150 150 269 269 269 F F F 17 340-359 150 150 150 269 269 269 F F F 18 360-379 145 145 145 262 262 262 F F F 19 380-399 144 144 144 261 261 261 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 150 150 150 269 269 269 F F F 23 460-479 148 148 148 265 265 265 F F F 24 480-499 150 150 150 269 269 269 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 145 145 145 262 262 262 F F F 27 540-559 150 150 150 269 269 269 F F F 28 560-579 148 148 148 267 267 267 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 150, Max flush: 269 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.744660217Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.753557607Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.753912511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.753964444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.753973973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.753986431Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Crucible a1feb75d-bd89-4067-92f8-f2002e1dbd34 has session id: e0409785-5dec-4b1f-b02c-7bb43c3d30c4","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754361813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Upstairs opts: Upstairs UUID: a1feb75d-bd89-4067-92f8-f2002e1dbd34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754398268Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Crucible stats registered with UUID: a1feb75d-bd89-4067-92f8-f2002e1dbd34","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754410506Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:22.754420755Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754561617Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754594662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754606331Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754772969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754791587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"a1feb75d-bd89-4067-92f8-f2002e1dbd34 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754824293Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754834541Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"0","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.75484513Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"1","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754855559Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"2","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754865847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754876436Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754887145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36465)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.754981002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33216)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.7549962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41710)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.755085399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.755702549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"0","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.755749633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"1","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.75577275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"2","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756108897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[0]R flush_numbers[0..12]: [269, 268, 266, 259, 259, 269, 269, 269, 262, 269, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756131834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[0]R generation[0..12]: [150, 149, 148, 142, 142, 150, 150, 150, 145, 150, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756145702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.75615773Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[1]R flush_numbers[0..12]: [269, 268, 266, 259, 259, 269, 269, 269, 262, 269, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756169069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[1]R generation[0..12]: [150, 149, 148, 142, 142, 150, 150, 150, 145, 150, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756180527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756192396Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[2]R flush_numbers[0..12]: [269, 268, 266, 259, 259, 269, 269, 269, 262, 269, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756203794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[2]R generation[0..12]: [150, 149, 148, 142, 142, 150, 150, 150, 145, 150, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756215063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756226711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Max found gen is 151","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.75623688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Generation requested: 152 >= found:151","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756247529Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756257547Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756267686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"":"downstairs","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756278155Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756287583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"0","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756297902Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"1","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756309501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"client":"2","":"downstairs","session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756319929Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756329668Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"a1feb75d-bd89-4067-92f8-f2002e1dbd34 is now active with session: e0409785-5dec-4b1f-b02c-7bb43c3d30c4","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756340487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756349845Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"session_id":"e0409785-5dec-4b1f-b02c-7bb43c3d30c4"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756359614Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Activated sub_volume a1feb75d-bd89-4067-92f8-f2002e1dbd34","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756368193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756376982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.756717158Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"task":"crutest"} 01/30 Write block 471 len 32768 data: 16 18 16 10 11 11 8 9 02/30 Write block 527 len 20480 data: 18 17 14 13 12 03/30 Write block 97 len 36864 data: 12 12 14 12 10 10 12 11 14 04/30 Read block 220 len 12288 05/30 Read block 84 len 28672 06/30 Read block 220 len 4096 07/30 Read block 4 len 4096 08/30 Read block 298 len 36864 09/30 Write block 564 len 20480 data: 15 15 14 11 10 10/30 Read block 257 len 16384 11/30 Write block 128 len 4096 data: 10 12/30 Write block 131 len 16384 data: 10 11 13 12 13/30 Read block 27 len 24576 14/30 Write block 310 len 20480 data: 15 18 14 13 13 15/30 Write block 192 len 4096 data: 13 16/30 Flush 17/30 Write block 535 len 16384 data: 13 15 16 17 18/30 Write block 360 len 24576 data: 21 17 22 21 23 20 19/30 Read block 496 len 36864 20/30 Read block 408 len 4096 21/30 Write block 482 len 20480 data: 10 10 8 8 8 22/30 Write block 20 len 20480 data: 13 13 9 11 12 23/30 Write block 380 len 20480 data: 12 15 16 13 12 24/30 Read block 296 len 36864 25/30 Flush 26/30 Write block 46 len 40960 data: 13 18 17 15 16 16 16 14 13 13 27/30 Read block 293 len 4096 28/30 Read block 59 len 24576 29/30 Write block 568 len 24576 data: 11 11 12 11 11 13 30/30 Write block 586 len 16384 data: 11 10 10 10 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 5 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 29 33 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:22.765312376Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:22.765534997Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1826,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 269 269 269 F F F 1 020-039 152 152 149 271 271 268 F F F <--- 2 040-059 148 148 148 266 266 266 T T F <--- 3 060-079 142 142 142 259 259 259 F F F 4 080-099 152 152 142 270 270 259 F F F <--- 5 100-119 152 152 150 270 270 269 F F F <--- 6 120-139 152 152 150 270 270 269 F F F <--- 7 140-159 150 150 150 269 269 269 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 152 152 150 270 270 269 F F F <--- 10 200-219 149 149 149 268 268 268 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 150 150 150 269 269 269 F F F 13 260-279 148 148 148 266 266 266 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 143 270 270 260 F F F <--- 16 320-339 150 150 150 269 269 269 F F F 17 340-359 150 150 150 269 269 269 F F F 18 360-379 152 152 145 271 271 262 F F F <--- 19 380-399 152 152 144 271 271 261 F F F <--- 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 150 150 150 269 269 269 F F F 23 460-479 152 152 148 270 270 265 F F T <--- 24 480-499 152 152 150 271 271 269 F F F <--- 25 500-519 144 144 144 261 261 261 F F F 26 520-539 152 152 145 271 271 262 F F F <--- 27 540-559 150 150 150 269 269 269 F F F 28 560-579 152 152 148 270 270 267 T T F <--- 29 580-599 142 142 142 259 259 259 T T F <--- Max gen: 152, Max flush: 271 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 269 269 269 F F F 1 020-039 152 152 152 271 271 271 F F F 2 040-059 153 153 153 272 272 272 F F F 3 060-079 142 142 142 259 259 259 F F F 4 080-099 152 152 152 270 270 270 F F F 5 100-119 152 152 152 270 270 270 F F F 6 120-139 152 152 152 270 270 270 F F F 7 140-159 150 150 150 269 269 269 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 152 152 152 270 270 270 F F F 10 200-219 149 149 149 268 268 268 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 150 150 150 269 269 269 F F F 13 260-279 148 148 148 266 266 266 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 152 270 270 270 F F F 16 320-339 150 150 150 269 269 269 F F F 17 340-359 150 150 150 269 269 269 F F F 18 360-379 152 152 152 271 271 271 F F F 19 380-399 152 152 152 271 271 271 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 150 150 150 269 269 269 F F F 23 460-479 152 152 152 270 270 270 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 152 152 152 271 271 271 F F F 27 540-559 150 150 150 269 269 269 F F F 28 560-579 153 153 153 272 272 272 F F F 29 580-599 153 153 153 272 272 272 F F F Max gen: 153, Max flush: 272 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.042864074Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.051916973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052242261Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052277387Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052287576Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052301104Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Crucible 8d80f4de-e5c1-4861-827e-34d4fdaa5987 has session id: ce3d65b1-39e9-48cb-afeb-511bb8fcd0de","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052669236Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Upstairs opts: Upstairs UUID: 8d80f4de-e5c1-4861-827e-34d4fdaa5987, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052691773Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Crucible stats registered with UUID: 8d80f4de-e5c1-4861-827e-34d4fdaa5987","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052703912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:37.05271471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052877389Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052896107Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.052907875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053108349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053124197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"8d80f4de-e5c1-4861-827e-34d4fdaa5987 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053166522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.05317764Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"0","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053189339Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"1","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053200717Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"2","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053211836Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053223504Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053236813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"ds_connection connected from Ok(127.0.0.1:36077)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053357177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"ds_connection connected from Ok(127.0.0.1:45365)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053387523Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"ds_connection connected from Ok(127.0.0.1:42389)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.053400832Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.05418228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"1","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054227915Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"0","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054250382Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"2","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054670147Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[0]R flush_numbers[0..12]: [269, 271, 272, 259, 270, 270, 270, 269, 262, 270, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054692195Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[0]R generation[0..12]: [150, 152, 153, 142, 152, 152, 152, 150, 145, 152, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054705063Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054718031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[1]R flush_numbers[0..12]: [269, 271, 272, 259, 270, 270, 270, 269, 262, 270, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.05473058Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[1]R generation[0..12]: [150, 152, 153, 142, 152, 152, 152, 150, 145, 152, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054743148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054756226Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[2]R flush_numbers[0..12]: [269, 271, 272, 259, 270, 270, 270, 269, 262, 270, 268, 268]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054771304Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[2]R generation[0..12]: [150, 152, 153, 142, 152, 152, 152, 150, 145, 152, 149, 149]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054783793Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054796811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.05480808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054819798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054830947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054842095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"":"downstairs","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054853534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054863732Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"0","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054875261Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"1","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054886629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"client":"2","":"downstairs","session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054898288Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054909006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"8d80f4de-e5c1-4861-827e-34d4fdaa5987 is now active with session: ce3d65b1-39e9-48cb-afeb-511bb8fcd0de","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054920795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054931104Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"session_id":"ce3d65b1-39e9-48cb-afeb-511bb8fcd0de"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054941922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Activated sub_volume 8d80f4de-e5c1-4861-827e-34d4fdaa5987","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.054952471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.05496201Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.055302126Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"task":"crutest"} 01/30 Write block 6 len 40960 data: 10 11 11 12 10 13 9 8 8 11 02/30 Read block 221 len 20480 03/30 Read block 5 len 20480 04/30 Read block 486 len 4096 05/30 Flush 06/30 Write block 270 len 20480 data: 10 13 13 14 16 07/30 Write block 213 len 36864 data: 6 5 9 12 14 18 19 18 17 08/30 Read block 472 len 20480 09/30 Write block 92 len 36864 data: 7 8 9 10 11 13 13 15 13 10/30 Read block 367 len 28672 11/30 Read block 312 len 16384 12/30 Flush 13/30 Write block 373 len 4096 data: 7 14/30 Write block 169 len 40960 data: 12 11 14 11 12 11 12 12 15 14 15/30 Write block 586 len 40960 data: 12 11 11 11 12 10 13 14 11 11 16/30 Flush 17/30 Write block 134 len 20480 data: 13 11 11 12 11 18/30 Write block 74 len 24576 data: 7 8 7 6 8 9 19/30 Write block 394 len 8192 data: 11 12 20/30 Write block 153 len 16384 data: 10 12 12 14 21/30 Read block 309 len 8192 22/30 Read block 27 len 12288 23/30 Read block 228 len 20480 24/30 Write block 468 len 28672 data: 16 15 16 17 19 17 11 25/30 Write block 195 len 32768 data: 7 8 7 8 9 11 10 13 26/30 Read block 385 len 32768 27/30 Write block 168 len 40960 data: 14 13 12 15 12 13 12 13 13 16 28/30 Write block 460 len 24576 data: 11 12 12 13 17 15 29/30 Write block 570 len 20480 data: 13 12 12 14 11 30/30 Write block 531 len 40960 data: 13 10 11 10 14 16 17 18 15 18 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 1 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 10 Done Sent Sent false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 4 27 33 Done 28 26 3 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:37.065299393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:37.065524663Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1835,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 150 273 273 269 F F T <--- 1 020-039 152 152 152 271 271 271 F F F 2 040-059 153 153 153 272 272 272 F F F 3 060-079 142 142 142 259 259 259 T T F <--- 4 080-099 154 154 152 274 274 270 F F F <--- 5 100-119 154 154 152 274 274 270 F F F <--- 6 120-139 152 152 152 270 270 270 T T F <--- 7 140-159 150 150 150 269 269 269 T T F <--- 8 160-179 154 154 145 275 275 262 T T F <--- 9 180-199 152 152 152 270 270 270 T T F <--- 10 200-219 154 154 149 274 274 268 T T F <--- 11 220-239 154 154 149 274 274 268 F F F <--- 12 240-259 150 150 150 269 269 269 F F F 13 260-279 154 154 148 274 274 266 F F F <--- 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 152 270 270 270 F F F 16 320-339 150 150 150 269 269 269 F F F 17 340-359 150 150 150 269 269 269 F F F 18 360-379 154 154 152 275 275 271 F F F <--- 19 380-399 152 152 152 271 271 271 T T F <--- 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 150 150 150 269 269 269 F F F 23 460-479 152 152 152 270 270 270 T T F <--- 24 480-499 152 152 152 271 271 271 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 152 152 152 271 271 271 T T F <--- 27 540-559 150 150 150 269 269 269 T T F <--- 28 560-579 153 153 153 272 272 272 T T F <--- 29 580-599 154 154 153 275 275 272 F F F <--- Max gen: 154, Max flush: 275 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 273 273 273 F F F 1 020-039 152 152 152 271 271 271 F F F 2 040-059 153 153 153 272 272 272 F F F 3 060-079 155 155 155 276 276 276 F F F 4 080-099 154 154 154 274 274 274 F F F 5 100-119 154 154 154 274 274 274 F F F 6 120-139 155 155 155 276 276 276 F F F 7 140-159 155 155 155 276 276 276 F F F 8 160-179 155 155 155 276 276 276 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 155 155 155 276 276 276 F F F 11 220-239 154 154 154 274 274 274 F F F 12 240-259 150 150 150 269 269 269 F F F 13 260-279 154 154 154 274 274 274 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 152 270 270 270 F F F 16 320-339 150 150 150 269 269 269 F F F 17 340-359 150 150 150 269 269 269 F F F 18 360-379 154 154 154 275 275 275 F F F 19 380-399 155 155 155 276 276 276 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 F F F 22 440-459 150 150 150 269 269 269 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 155 155 155 276 276 276 F F F 27 540-559 155 155 155 276 276 276 F F F 28 560-579 155 155 155 276 276 276 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 155, Max flush: 276 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.359150623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.368439782Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.368768819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.368803515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.368813823Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.368827461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Crucible 69fd7aef-0175-4b15-8307-942579e9fbc6 has session id: 7c069372-c9e2-4443-abc1-4e8bfb0672be","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369216751Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Upstairs opts: Upstairs UUID: 69fd7aef-0175-4b15-8307-942579e9fbc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369312109Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Crucible stats registered with UUID: 69fd7aef-0175-4b15-8307-942579e9fbc6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369340055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:51.369352144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369517782Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369552878Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369566006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369678711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369699279Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"69fd7aef-0175-4b15-8307-942579e9fbc6 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369730295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369741313Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"0","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369752772Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"1","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.36976408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"2","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369775629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369801046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369812844Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"ds_connection connected from Ok(127.0.0.1:53909)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369859858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"ds_connection connected from Ok(127.0.0.1:58225)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369872506Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"ds_connection connected from Ok(127.0.0.1:52914)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.369893364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.370740644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"0","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.37076963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"1","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.370784428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"2","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371177847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[0]R flush_numbers[0..12]: [273, 271, 272, 276, 274, 274, 276, 276, 276, 276, 276, 274]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371207704Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[0]R generation[0..12]: [154, 152, 153, 155, 154, 154, 155, 155, 155, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371220932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.37123386Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[1]R flush_numbers[0..12]: [273, 271, 272, 276, 274, 274, 276, 276, 276, 276, 276, 274]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371246399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[1]R generation[0..12]: [154, 152, 153, 155, 154, 154, 155, 155, 155, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371258927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371272035Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[2]R flush_numbers[0..12]: [273, 271, 272, 276, 274, 274, 276, 276, 276, 276, 276, 274]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371284724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[2]R generation[0..12]: [154, 152, 153, 155, 154, 154, 155, 155, 155, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371297242Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.37131008Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371321319Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371334347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371345476Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371356664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"":"downstairs","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371368223Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371378772Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"0","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.37139007Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"1","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371401349Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"client":"2","":"downstairs","session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371412947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371423706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"69fd7aef-0175-4b15-8307-942579e9fbc6 is now active with session: 7c069372-c9e2-4443-abc1-4e8bfb0672be","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371435524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371446153Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"session_id":"7c069372-c9e2-4443-abc1-4e8bfb0672be"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371456901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Activated sub_volume 69fd7aef-0175-4b15-8307-942579e9fbc6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.37146647Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.371476309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.37185326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"task":"crutest"} 01/30 Write block 134 len 28672 data: 14 12 12 13 12 13 11 02/30 Read block 531 len 12288 03/30 Read block 519 len 4096 04/30 Read block 227 len 24576 05/30 Read block 57 len 12288 06/30 Flush 07/30 Write block 271 len 32768 data: 14 14 15 17 17 16 12 11 08/30 Write block 572 len 20480 data: 13 15 12 13 14 09/30 Write block 353 len 40960 data: 14 16 17 14 13 16 18 22 18 23 10/30 Write block 450 len 28672 data: 10 8 9 8 9 6 6 11/30 Write block 338 len 32768 data: 6 7 9 9 8 8 9 12 12/30 Write block 269 len 24576 data: 9 11 15 15 16 18 13/30 Read block 327 len 36864 14/30 Read block 236 len 36864 15/30 Write block 535 len 8192 data: 15 17 16/30 Read block 571 len 32768 17/30 Read block 271 len 4096 18/30 Read block 245 len 32768 19/30 Read block 94 len 4096 20/30 Write block 378 len 32768 data: 9 12 13 16 17 14 13 14 21/30 Write block 435 len 28672 data: 12 14 15 14 13 14 14 22/30 Write block 42 len 20480 data: 13 13 15 14 14 23/30 Read block 394 len 24576 24/30 Read block 67 len 20480 25/30 Write block 240 len 8192 data: 9 10 26/30 Read block 193 len 24576 27/30 Read block 384 len 20480 28/30 Read block 299 len 4096 29/30 Write block 125 len 24576 data: 14 12 9 11 12 12 30/30 Read block 421 len 24576 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 3 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:04:51.381635175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:04:51.381849777Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1844,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 273 273 273 F F F 1 020-039 152 152 152 271 271 271 F F F 2 040-059 153 153 153 272 272 272 T T F <--- 3 060-079 155 155 155 276 276 276 F F F 4 080-099 154 154 154 274 274 274 F F F 5 100-119 154 154 154 274 274 274 F F F 6 120-139 156 156 155 277 277 276 T T T <--- 7 140-159 156 156 155 277 277 276 F F T <--- 8 160-179 155 155 155 276 276 276 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 155 155 155 276 276 276 F F F 11 220-239 154 154 154 274 274 274 F F F 12 240-259 150 150 150 269 269 269 T T F <--- 13 260-279 154 154 154 274 274 274 T T F <--- 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 152 270 270 270 F F F 16 320-339 150 150 150 269 269 269 T T F <--- 17 340-359 150 150 150 269 269 269 T T F <--- 18 360-379 154 154 154 275 275 275 T T F <--- 19 380-399 155 155 155 276 276 276 T T F <--- 20 400-419 144 144 144 261 261 261 F F F 21 420-439 144 144 144 261 261 261 T T F <--- 22 440-459 150 150 150 269 269 269 T T F <--- 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 155 155 155 276 276 276 T T F <--- 27 540-559 155 155 155 276 276 276 F F F 28 560-579 155 155 155 276 276 276 T T F <--- 29 580-599 154 154 154 275 275 275 F F F Max gen: 156, Max flush: 277 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 273 273 273 F F F 1 020-039 152 152 152 271 271 271 F F F 2 040-059 157 157 157 278 278 278 F F F 3 060-079 155 155 155 276 276 276 F F F 4 080-099 154 154 154 274 274 274 F F F 5 100-119 154 154 154 274 274 274 F F F 6 120-139 157 157 157 278 278 278 F F F 7 140-159 156 156 156 277 277 277 F F F 8 160-179 155 155 155 276 276 276 F F F 9 180-199 155 155 155 276 276 276 F F F 10 200-219 155 155 155 276 276 276 F F F 11 220-239 154 154 154 274 274 274 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 157 157 157 278 278 278 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 152 270 270 270 F F F 16 320-339 157 157 157 278 278 278 F F F 17 340-359 157 157 157 278 278 278 F F F 18 360-379 157 157 157 278 278 278 F F F 19 380-399 157 157 157 278 278 278 F F F 20 400-419 144 144 144 261 261 261 F F F 21 420-439 157 157 157 278 278 278 F F F 22 440-459 157 157 157 278 278 278 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 157 157 157 278 278 278 F F F 27 540-559 155 155 155 276 276 276 F F F 28 560-579 157 157 157 278 278 278 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 157, Max flush: 278 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.691230179Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.699861913Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700205428Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700242073Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700251802Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700264481Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Crucible 37ad5b09-ebed-43c6-85a3-d2b5c801b6b2 has session id: efda987d-f370-4ad6-81f6-b505b6177959","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.70065541Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Upstairs opts: Upstairs UUID: 37ad5b09-ebed-43c6-85a3-d2b5c801b6b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700719502Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Crucible stats registered with UUID: 37ad5b09-ebed-43c6-85a3-d2b5c801b6b2","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.70073266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:05.700742839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.70088492Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700902478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.700913557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701106872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701124329Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"37ad5b09-ebed-43c6-85a3-d2b5c801b6b2 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701159675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701172273Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"0","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701182952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"1","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.70119357Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"2","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701204279Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701215408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701226386Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"ds_connection connected from Ok(127.0.0.1:33358)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701258062Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"2","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"ds_connection connected from Ok(127.0.0.1:51069)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701288348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"0","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"ds_connection connected from Ok(127.0.0.1:50015)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.701300237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"io task","client":"1","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702026653Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"0","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702057129Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"2","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702070117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"1","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702455517Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[0]R flush_numbers[0..12]: [273, 271, 278, 276, 274, 274, 278, 277, 276, 276, 276, 274]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702479594Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[0]R generation[0..12]: [154, 152, 157, 155, 154, 154, 157, 156, 155, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702492072Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702506341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[1]R flush_numbers[0..12]: [273, 271, 278, 276, 274, 274, 278, 277, 276, 276, 276, 274]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702517969Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[1]R generation[0..12]: [154, 152, 157, 155, 154, 154, 157, 156, 155, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702529578Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702541586Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[2]R flush_numbers[0..12]: [273, 271, 278, 276, 274, 274, 278, 277, 276, 276, 276, 274]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702553295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[2]R generation[0..12]: [154, 152, 157, 155, 154, 154, 157, 156, 155, 155, 155, 154]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702564933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702576852Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.70258726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702598329Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702608607Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702618926Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"":"downstairs","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702629735Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702639323Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"0","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702650012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"1","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702660591Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"client":"2","":"downstairs","session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702672759Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702682828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"37ad5b09-ebed-43c6-85a3-d2b5c801b6b2 is now active with session: efda987d-f370-4ad6-81f6-b505b6177959","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702693746Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702703295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"session_id":"efda987d-f370-4ad6-81f6-b505b6177959"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702713444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Activated sub_volume 37ad5b09-ebed-43c6-85a3-d2b5c801b6b2","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702722203Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.702731092Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.703088265Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"task":"crutest"} 01/30 Flush 02/30 Write block 367 len 20480 data: 19 17 16 14 10 03/30 Write block 180 len 36864 data: 10 11 12 17 15 14 16 12 10 04/30 Read block 480 len 36864 05/30 Write block 98 len 36864 data: 14 16 14 11 11 13 12 15 16 06/30 Read block 210 len 8192 07/30 Read block 491 len 32768 08/30 Write block 332 len 16384 data: 13 13 11 10 09/30 Read block 105 len 28672 10/30 Read block 454 len 16384 11/30 Write block 226 len 24576 data: 15 16 13 9 7 7 12/30 Write block 11 len 28672 data: 14 10 9 9 12 12 13 13/30 Read block 0 len 32768 14/30 Write block 563 len 16384 data: 12 16 16 15 15/30 Read block 545 len 8192 16/30 Write block 190 len 8192 data: 8 12 17/30 Read block 122 len 4096 18/30 Write block 508 len 20480 data: 12 11 11 16 16 19/30 Write block 73 len 40960 data: 7 8 9 8 7 9 10 9 8 10 20/30 Write block 190 len 4096 data: 9 21/30 Write block 47 len 24576 data: 19 18 16 17 17 17 22/30 Write block 443 len 28672 data: 14 13 12 13 10 10 11 23/30 Flush 24/30 Write block 302 len 16384 data: 11 11 12 11 25/30 Write block 145 len 8192 data: 12 11 26/30 Read block 125 len 28672 27/30 Write block 506 len 36864 data: 13 13 13 12 12 17 17 14 12 28/30 Read block 43 len 40960 29/30 Write block 417 len 28672 data: 13 14 11 10 10 10 9 30/30 Read block 271 len 4096 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:05.712503898Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:05.712726839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1853,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 154 158 280 273 280 F F F <--- 1 020-039 152 152 152 271 271 271 F F F 2 040-059 158 157 158 280 278 280 F F F <--- 3 060-079 158 155 158 280 276 280 F F F <--- 4 080-099 158 154 158 280 274 280 F F F <--- 5 100-119 158 154 158 280 274 280 F F F <--- 6 120-139 157 157 157 278 278 278 F F F 7 140-159 156 156 156 277 277 277 T F T <--- 8 160-179 155 155 155 276 276 276 F F F 9 180-199 158 155 158 280 276 280 F F F <--- 10 200-219 155 155 155 276 276 276 F F F 11 220-239 158 154 158 280 274 280 F F F <--- 12 240-259 157 157 157 278 278 278 F F F 13 260-279 157 157 157 278 278 278 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 152 152 152 270 270 270 T F T <--- 16 320-339 158 157 158 280 278 280 F F F <--- 17 340-359 157 157 157 278 278 278 F F F 18 360-379 158 157 158 280 278 280 F F F <--- 19 380-399 157 157 157 278 278 278 F F F 20 400-419 144 144 144 261 261 261 T F T <--- 21 420-439 157 157 157 278 278 278 T F T <--- 22 440-459 158 157 158 280 278 280 F F F <--- 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 158 144 158 280 261 280 T F T <--- 26 520-539 157 157 157 278 278 278 F F F 27 540-559 155 155 155 276 276 276 F F F 28 560-579 158 157 158 280 278 280 F F F <--- 29 580-599 154 154 154 275 275 275 F F F Max gen: 158, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 280 280 280 F F F 1 020-039 152 152 152 271 271 271 F F F 2 040-059 158 158 158 280 280 280 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 158 158 158 280 280 280 F F F 6 120-139 157 157 157 278 278 278 F F F 7 140-159 159 159 159 281 281 281 F F F 8 160-179 155 155 155 276 276 276 F F F 9 180-199 158 158 158 280 280 280 F F F 10 200-219 155 155 155 276 276 276 F F F 11 220-239 158 158 158 280 280 280 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 157 157 157 278 278 278 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 159 159 159 281 281 281 F F F 16 320-339 158 158 158 280 280 280 F F F 17 340-359 157 157 157 278 278 278 F F F 18 360-379 158 158 158 280 280 280 F F F 19 380-399 157 157 157 278 278 278 F F F 20 400-419 159 159 159 281 281 281 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 159 159 159 281 281 281 F F F 26 520-539 157 157 157 278 278 278 F F F 27 540-559 155 155 155 276 276 276 F F F 28 560-579 158 158 158 280 280 280 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 159, Max flush: 281 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:19.998985031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:20.00656728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.006619133Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.013920689Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014191224Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01422527Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014234788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014250106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"Crucible d4ca4613-ad8d-45c2-935b-f1afc9e583ab has session id: a5d6bb06-057c-4b9e-8651-dda23a08e57b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014588763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Upstairs opts: Upstairs UUID: d4ca4613-ad8d-45c2-935b-f1afc9e583ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01460881Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Crucible stats registered with UUID: d4ca4613-ad8d-45c2-935b-f1afc9e583ab","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014623058Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:20.014632787Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01476124Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014778788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014789387Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014969733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.014987521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"d4ca4613-ad8d-45c2-935b-f1afc9e583ab active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015012798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015022796Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"0","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015033115Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"1","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015043444Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"2","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015053782Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015114085Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015157469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"ds_connection connected from Ok(127.0.0.1:34362)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015182866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56232)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015280283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36480)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01530762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015808225Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"2","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015828662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"1","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.015956766Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"0","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016228031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[0]R flush_numbers[0..12]: [280, 271, 280, 280, 280, 280, 278, 281, 276, 280, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016258197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[0]R generation[0..12]: [158, 152, 158, 158, 158, 158, 157, 159, 155, 158, 155, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016270135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016281944Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[1]R flush_numbers[0..12]: [280, 271, 280, 280, 280, 280, 278, 281, 276, 280, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016293142Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[1]R generation[0..12]: [158, 152, 158, 158, 158, 158, 157, 159, 155, 158, 155, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016304341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016318989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[2]R flush_numbers[0..12]: [280, 271, 280, 280, 280, 280, 278, 281, 276, 280, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016330277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[2]R generation[0..12]: [158, 152, 158, 158, 158, 158, 157, 159, 155, 158, 155, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016341416Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016353014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016362983Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016373362Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01638331Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016393429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"":"downstairs","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016403988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016413417Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"0","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016423925Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"1","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016434354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"client":"2","":"downstairs","session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016444962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016454691Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"d4ca4613-ad8d-45c2-935b-f1afc9e583ab is now active with session: a5d6bb06-057c-4b9e-8651-dda23a08e57b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01646518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016475808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"session_id":"a5d6bb06-057c-4b9e-8651-dda23a08e57b"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016485567Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"Activated sub_volume d4ca4613-ad8d-45c2-935b-f1afc9e583ab","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016494186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.016508064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.01677002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} 01/30 Write block 42 len 40960 data: 14 14 16 15 15 20 19 17 18 18 02/30 Flush 03/30 Write block 274 len 12288 data: 19 18 17 04/30 Read block 459 len 36864 05/30 Read block 438 len 24576 06/30 Read block 253 len 24576 07/30 Read block 299 len 40960 08/30 Write block 188 len 28672 data: 11 10 10 13 14 10 5 09/30 Write block 19 len 36864 data: 14 14 14 10 12 13 11 12 10 10/30 Write block 7 len 32768 data: 12 12 13 11 15 11 10 10 11/30 Write block 156 len 32768 data: 15 15 16 17 16 15 15 12 12/30 Read block 495 len 40960 13/30 Read block 185 len 24576 14/30 Write block 575 len 8192 data: 14 15 15/30 Write block 534 len 12288 data: 11 16 18 16/30 Write block 503 len 24576 data: 12 14 11 14 14 14 17/30 Write block 128 len 24576 data: 12 13 13 11 12 14 18/30 Read block 285 len 16384 19/30 Write block 346 len 8192 data: 13 12 20/30 Write block 261 len 40960 data: 7 9 9 7 6 8 11 11 10 12 21/30 Write block 312 len 36864 data: 15 14 14 12 11 12 7 4 7 22/30 Read block 158 len 20480 23/30 Read block 413 len 28672 24/30 Read block 466 len 40960 25/30 Read block 28 len 8192 26/30 Read block 497 len 12288 27/30 Read block 325 len 16384 28/30 Read block 333 len 32768 29/30 Write block 268 len 28672 data: 12 11 13 16 16 17 20 30/30 Write block 358 len 32768 data: 17 19 23 19 24 22 24 21 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 2 31 Done 1 26 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:20.02589965Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:20.026120491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1862,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 280 280 280 F T T <--- 1 020-039 152 152 152 271 271 271 F T T <--- 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 158 158 158 280 280 280 F F F 6 120-139 157 157 157 278 278 278 F T T <--- 7 140-159 159 159 159 281 281 281 F T T <--- 8 160-179 155 155 155 276 276 276 F T T <--- 9 180-199 158 158 158 280 280 280 F T T <--- 10 200-219 155 155 155 276 276 276 F F F 11 220-239 158 158 158 280 280 280 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 157 157 157 278 278 278 T T T 14 280-299 149 149 149 268 268 268 F F F 15 300-319 159 159 159 281 281 281 F T T <--- 16 320-339 158 158 158 280 280 280 F T T <--- 17 340-359 157 157 157 278 278 278 F T T <--- 18 360-379 158 158 158 280 280 280 F T T <--- 19 380-399 157 157 157 278 278 278 F F F 20 400-419 159 159 159 281 281 281 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 159 159 159 281 281 281 F T T <--- 26 520-539 157 157 157 278 278 278 F T T <--- 27 540-559 155 155 155 276 276 276 F F F 28 560-579 158 158 158 280 280 280 F T T <--- 29 580-599 154 154 154 275 275 275 F F F Max gen: 160, Max flush: 282 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 264,268d263 < 6, < 8, < 8, < 6, < 5, 270,271d264 < 10, < 10, 272a266,269 > 9, > 7, > 6, > 8, 274,275c271,273 < 15, < 15, --- > 12, > 11, > 13, 277c275,277 < 19, --- > 16, > 17, > 20, diff found after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 161 161 161 283 283 283 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 158 158 158 280 280 280 F F F 6 120-139 161 161 161 283 283 283 F F F 7 140-159 161 161 161 283 283 283 F F F 8 160-179 161 161 161 283 283 283 F F F 9 180-199 161 161 161 283 283 283 F F F 10 200-219 155 155 155 276 276 276 F F F 11 220-239 158 158 158 280 280 280 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 161 161 161 283 283 283 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 161 161 161 283 283 283 F F F 16 320-339 161 161 161 283 283 283 F F F 17 340-359 161 161 161 283 283 283 F F F 18 360-379 161 161 161 283 283 283 F F F 19 380-399 157 157 157 278 278 278 F F F 20 400-419 159 159 159 281 281 281 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 155 155 155 276 276 276 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 161 161 161 283 283 283 F F F 26 520-539 161 161 161 283 283 283 F F F 27 540-559 155 155 155 276 276 276 F F F 28 560-579 161 161 161 283 283 283 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 161, Max flush: 283 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.310279715Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.318577132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.318874294Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.318905769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.318914498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.318926377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Crucible 61dbeb86-2534-4fb2-a8cb-d19134e5b863 has session id: 4fa8f710-19fc-4b11-a69c-3a758c0876d8","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319323265Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Upstairs opts: Upstairs UUID: 61dbeb86-2534-4fb2-a8cb-d19134e5b863, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319347542Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Crucible stats registered with UUID: 61dbeb86-2534-4fb2-a8cb-d19134e5b863","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319360131Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:34.319369729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319499753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.31951733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319528289Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319703486Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319727783Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"61dbeb86-2534-4fb2-a8cb-d19134e5b863 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319737202Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319746681Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"0","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319757119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"1","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319767328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"2","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319777467Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319810952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319833899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36420)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319893162Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55740)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319918978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61339)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.319933087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.320665132Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"2","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.320684999Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"0","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.320696678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"1","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321042733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[0]R flush_numbers[0..12]: [283, 283, 282, 280, 280, 280, 283, 283, 283, 283, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.32106391Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[0]R generation[0..12]: [161, 161, 160, 158, 158, 158, 161, 161, 161, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321075459Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321087277Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[1]R flush_numbers[0..12]: [283, 283, 282, 280, 280, 280, 283, 283, 283, 283, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321098536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[1]R generation[0..12]: [161, 161, 160, 158, 158, 158, 161, 161, 161, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321109694Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321121173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[2]R flush_numbers[0..12]: [283, 283, 282, 280, 280, 280, 283, 283, 283, 283, 276, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321132421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[2]R generation[0..12]: [161, 161, 160, 158, 158, 158, 161, 161, 161, 161, 155, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.32114353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321154899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321166807Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321177186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321187064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321196993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"":"downstairs","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321207182Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321216441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"0","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321226499Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"1","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321236598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"client":"2","":"downstairs","session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321246967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321256515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"61dbeb86-2534-4fb2-a8cb-d19134e5b863 is now active with session: 4fa8f710-19fc-4b11-a69c-3a758c0876d8","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321267014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321276203Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"session_id":"4fa8f710-19fc-4b11-a69c-3a758c0876d8"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321285812Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Activated sub_volume 61dbeb86-2534-4fb2-a8cb-d19134e5b863","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.32129927Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321308009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.321568745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"task":"crutest"} 01/30 Write block 217 len 12288 data: 15 19 20 02/30 Read block 109 len 16384 03/30 Read block 560 len 40960 04/30 Read block 357 len 40960 05/30 Read block 357 len 12288 06/30 Flush 07/30 Write block 213 len 20480 data: 7 6 10 13 16 08/30 Read block 17 len 36864 09/30 Write block 395 len 8192 data: 13 13 10/30 Read block 523 len 4096 11/30 Write block 393 len 28672 data: 12 12 14 14 14 11 8 12/30 Write block 361 len 28672 data: 20 25 23 25 22 21 20 13/30 Read block 432 len 12288 14/30 Write block 24 len 32768 data: 14 12 13 11 10 10 8 7 15/30 Read block 204 len 24576 16/30 Write block 417 len 12288 data: 14 15 12 17/30 Write block 437 len 36864 data: 16 15 14 15 15 16 15 14 13 18/30 Read block 291 len 40960 19/30 Flush 20/30 Flush 21/30 Write block 419 len 24576 data: 13 11 11 11 10 7 22/30 Read block 302 len 12288 23/30 Write block 532 len 8192 data: 11 12 24/30 Write block 464 len 28672 data: 18 16 12 15 17 16 17 25/30 Read block 384 len 12288 26/30 Read block 111 len 40960 27/30 Write block 519 len 36864 data: 15 18 16 18 16 14 15 15 19 28/30 Write block 128 len 16384 data: 13 14 14 12 29/30 Write block 532 len 36864 data: 12 13 12 17 19 18 19 16 19 30/30 Read block 32 len 28672 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:34.33080733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:34.331015503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1874,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 162 161 162 285 283 285 F F F <--- 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 158 158 158 280 280 280 F F F 6 120-139 161 161 161 283 283 283 T F T <--- 7 140-159 161 161 161 283 283 283 F F F 8 160-179 161 161 161 283 283 283 F F F 9 180-199 161 161 161 283 283 283 F F F 10 200-219 162 155 162 285 276 285 F T F <--- 11 220-239 158 158 158 280 280 280 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 161 161 161 283 283 283 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 161 161 161 283 283 283 F F F 16 320-339 161 161 161 283 283 283 F F F 17 340-359 161 161 161 283 283 283 F F F 18 360-379 162 161 162 285 283 285 F F F <--- 19 380-399 162 157 162 285 278 285 F F F <--- 20 400-419 162 159 162 285 281 285 T F T <--- 21 420-439 162 159 162 285 281 285 T F T <--- 22 440-459 162 158 162 285 280 285 F F F <--- 23 460-479 155 155 155 276 276 276 T F T <--- 24 480-499 152 152 152 271 271 271 F F F 25 500-519 161 161 161 283 283 283 T F T <--- 26 520-539 161 161 161 283 283 283 T F T <--- 27 540-559 155 155 155 276 276 276 T F T <--- 28 560-579 161 161 161 283 283 283 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 162, Max flush: 285 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 162 162 162 285 285 285 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 158 158 158 280 280 280 F F F 6 120-139 163 163 163 286 286 286 F F F 7 140-159 161 161 161 283 283 283 F F F 8 160-179 161 161 161 283 283 283 F F F 9 180-199 161 161 161 283 283 283 F F F 10 200-219 162 162 162 285 285 285 F F F 11 220-239 158 158 158 280 280 280 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 161 161 161 283 283 283 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 161 161 161 283 283 283 F F F 16 320-339 161 161 161 283 283 283 F F F 17 340-359 161 161 161 283 283 283 F F F 18 360-379 162 162 162 285 285 285 F F F 19 380-399 162 162 162 285 285 285 F F F 20 400-419 163 163 163 286 286 286 F F F 21 420-439 163 163 163 286 286 286 F F F 22 440-459 162 162 162 285 285 285 F F F 23 460-479 163 163 163 286 286 286 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 163 163 163 286 286 286 F F F 27 540-559 163 163 163 286 286 286 F F F 28 560-579 161 161 161 283 283 283 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 163, Max flush: 286 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.603446624Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.612343144Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.612660113Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.612690399Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.612699867Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.612712266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Crucible 231ceceb-61f4-4b9a-8f56-22c41925f9aa has session id: 77f52580-46c0-494c-bf2e-d6598cc0264f","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613082908Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Upstairs opts: Upstairs UUID: 231ceceb-61f4-4b9a-8f56-22c41925f9aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613114644Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Crucible stats registered with UUID: 231ceceb-61f4-4b9a-8f56-22c41925f9aa","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613126702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:48.613136871Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613274593Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613306129Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613317897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613466528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613485636Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"231ceceb-61f4-4b9a-8f56-22c41925f9aa active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613519141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.61352934Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"0","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613539979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"1","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613550537Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"2","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613561006Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613571955Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613582583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39724)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613648975Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53981)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613660863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56525)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.613777538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614404377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"2","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614443242Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"1","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.61445525Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"0","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614854909Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[0]R flush_numbers[0..12]: [283, 285, 282, 280, 280, 280, 286, 283, 283, 283, 285, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614885435Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[0]R generation[0..12]: [161, 162, 160, 158, 158, 158, 163, 161, 161, 161, 162, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614897723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614909762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[1]R flush_numbers[0..12]: [283, 285, 282, 280, 280, 280, 286, 283, 283, 283, 285, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.61492141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[1]R generation[0..12]: [161, 162, 160, 158, 158, 158, 163, 161, 161, 161, 162, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614932899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614944767Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[2]R flush_numbers[0..12]: [283, 285, 282, 280, 280, 280, 286, 283, 283, 283, 285, 280]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614956376Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[2]R generation[0..12]: [161, 162, 160, 158, 158, 158, 163, 161, 161, 161, 162, 158]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614967754Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614979633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.614989901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.6150005Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615011248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615021487Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"":"downstairs","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615032026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615041485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"0","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615053563Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"1","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615063992Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"client":"2","":"downstairs","session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.61507474Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615084729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"231ceceb-61f4-4b9a-8f56-22c41925f9aa is now active with session: 77f52580-46c0-494c-bf2e-d6598cc0264f","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615095478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615104976Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"session_id":"77f52580-46c0-494c-bf2e-d6598cc0264f"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615114955Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Activated sub_volume 231ceceb-61f4-4b9a-8f56-22c41925f9aa","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615123684Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615132583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.615482218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"task":"crutest"} 01/30 Write block 467 len 36864 data: 16 18 17 18 18 20 18 12 12 02/30 Write block 374 len 28672 data: 7 5 6 8 10 13 14 03/30 Flush 04/30 Write block 315 len 40960 data: 13 12 13 8 5 8 7 6 8 7 05/30 Read block 109 len 24576 06/30 Read block 197 len 12288 07/30 Read block 300 len 4096 08/30 Read block 442 len 32768 09/30 Flush 10/30 Write block 175 len 20480 data: 14 14 17 15 10 11/30 Write block 101 len 4096 data: 12 12/30 Write block 118 len 24576 data: 10 10 10 13 12 12 13/30 Write block 525 len 28672 data: 16 16 20 18 15 14 14 14/30 Read block 380 len 20480 15/30 Read block 479 len 20480 16/30 Read block 82 len 24576 17/30 Read block 211 len 12288 18/30 Write block 433 len 8192 data: 11 14 19/30 Write block 91 len 36864 data: 8 8 9 10 11 12 14 15 17 20/30 Read block 354 len 36864 21/30 Read block 173 len 20480 22/30 Read block 339 len 32768 23/30 Flush 24/30 Write block 537 len 36864 data: 19 20 17 20 19 19 15 12 12 25/30 Write block 106 len 24576 data: 17 14 13 14 12 10 26/30 Write block 348 len 24576 data: 11 13 15 14 13 15 27/30 Read block 207 len 36864 28/30 Write block 221 len 4096 data: 18 29/30 Write block 435 len 16384 data: 13 15 17 16 30/30 Write block 268 len 24576 data: 11 10 12 16 16 17 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 19 2 23 Done 19 2 19 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1008)) Some(JobId(1022)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:05:48.624609447Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:05:48.62481652Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1883,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 162 162 162 285 285 285 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 164 158 164 289 280 289 F F F <--- 5 100-119 164 158 164 289 280 289 T T T <--- 6 120-139 164 163 164 289 286 289 F F F <--- 7 140-159 161 161 161 283 283 283 F F F 8 160-179 164 161 164 289 283 289 F T F <--- 9 180-199 161 161 161 283 283 283 F F F 10 200-219 162 162 162 285 285 285 F F F 11 220-239 158 158 158 280 280 280 T F T <--- 12 240-259 157 157 157 278 278 278 F F F 13 260-279 161 161 161 283 283 283 T F T <--- 14 280-299 149 149 149 268 268 268 F F F 15 300-319 164 164 164 288 288 288 F F F 16 320-339 164 164 164 288 288 288 F F F 17 340-359 161 161 161 283 283 283 T F T <--- 18 360-379 164 164 164 287 287 287 F F F 19 380-399 164 164 164 287 287 287 F F F 20 400-419 163 163 163 286 286 286 F F F 21 420-439 164 163 164 289 286 289 T F T <--- 22 440-459 162 162 162 285 285 285 F F F 23 460-479 164 164 164 287 287 287 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 164 163 164 289 286 289 T F T <--- 27 540-559 163 163 163 286 286 286 T F T <--- 28 560-579 161 161 161 283 283 283 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 164, Max flush: 289 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 162 162 162 285 285 285 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 165 165 165 290 290 290 F F F 6 120-139 164 164 164 289 289 289 F F F 7 140-159 161 161 161 283 283 283 F F F 8 160-179 164 164 164 289 289 289 F F F 9 180-199 161 161 161 283 283 283 F F F 10 200-219 162 162 162 285 285 285 F F F 11 220-239 165 165 165 290 290 290 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 165 165 165 290 290 290 F F F 14 280-299 149 149 149 268 268 268 F F F 15 300-319 164 164 164 288 288 288 F F F 16 320-339 164 164 164 288 288 288 F F F 17 340-359 165 165 165 290 290 290 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 164 164 164 287 287 287 F F F 20 400-419 163 163 163 286 286 286 F F F 21 420-439 165 165 165 290 290 290 F F F 22 440-459 162 162 162 285 285 285 F F F 23 460-479 164 164 164 287 287 287 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 165 165 165 290 290 290 F F F 27 540-559 165 165 165 290 290 290 F F F 28 560-579 161 161 161 283 283 283 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 165, Max flush: 290 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.889465728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898144656Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898453176Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898485522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.89849483Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898507199Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Crucible 4c72a7e5-e7ed-4b4c-b648-f8cc26211a76 has session id: cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898873311Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Upstairs opts: Upstairs UUID: 4c72a7e5-e7ed-4b4c-b648-f8cc26211a76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898893259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Crucible stats registered with UUID: 4c72a7e5-e7ed-4b4c-b648-f8cc26211a76","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.898906707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:02.898916526Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899086984Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.89911933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899131418Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899278939Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899292817Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"4c72a7e5-e7ed-4b4c-b648-f8cc26211a76 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899318924Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899329392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"0","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899339911Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"1","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.89935049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"2","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899360958Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899371577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899382186Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48511)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.89950514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40118)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899585739Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47179)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.899608436Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900127029Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"0","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.90020046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"2","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900234945Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"1","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900545075Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[0]R flush_numbers[0..12]: [283, 285, 282, 280, 289, 290, 289, 283, 289, 283, 285, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900577521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[0]R generation[0..12]: [161, 162, 160, 158, 164, 165, 164, 161, 164, 161, 162, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900589909Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900601968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[1]R flush_numbers[0..12]: [283, 285, 282, 280, 289, 290, 289, 283, 289, 283, 285, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900613556Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[1]R generation[0..12]: [161, 162, 160, 158, 164, 165, 164, 161, 164, 161, 162, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900625055Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900636973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[2]R flush_numbers[0..12]: [283, 285, 282, 280, 289, 290, 289, 283, 289, 283, 285, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900648552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[2]R generation[0..12]: [161, 162, 160, 158, 164, 165, 164, 161, 164, 161, 162, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.90066245Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900674528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900684897Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900695536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900705804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900716143Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"":"downstairs","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900726912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.90073645Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"0","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900746899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"1","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900757458Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"client":"2","":"downstairs","session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900768206Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900778295Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"4c72a7e5-e7ed-4b4c-b648-f8cc26211a76 is now active with session: cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900789194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900798752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"session_id":"cc18362d-8ac5-4a6e-9c0c-1bd52899e1b7"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900808721Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Activated sub_volume 4c72a7e5-e7ed-4b4c-b648-f8cc26211a76","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.90081742Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.900827469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.901167205Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"task":"crutest"} 01/30 Write block 297 len 16384 data: 7 5 7 10 02/30 Write block 570 len 36864 data: 14 13 14 16 13 15 16 15 14 03/30 Read block 535 len 8192 04/30 Write block 295 len 20480 data: 10 9 8 6 8 05/30 Write block 511 len 20480 data: 18 18 15 13 9 06/30 Read block 241 len 4096 07/30 Flush 08/30 Write block 552 len 24576 data: 8 9 9 9 10 8 09/30 Write block 198 len 40960 data: 9 10 12 11 14 11 13 11 9 13 10/30 Read block 76 len 16384 11/30 Write block 439 len 20480 data: 15 16 16 17 16 12/30 Write block 477 len 4096 data: 9 13/30 Write block 275 len 36864 data: 19 18 13 12 10 12 10 8 10 14/30 Write block 416 len 8192 data: 10 15 15/30 Write block 561 len 28672 data: 16 16 13 17 17 16 12 16/30 Write block 302 len 28672 data: 12 12 13 12 15 15 15 17/30 Flush 18/30 Write block 31 len 24576 data: 8 9 9 9 9 11 19/30 Write block 158 len 16384 data: 17 18 17 16 20/30 Write block 531 len 4096 data: 15 21/30 Write block 565 len 40960 data: 18 17 13 12 12 15 14 15 17 14 22/30 Write block 281 len 32768 data: 11 9 11 8 12 13 15 17 23/30 Write block 277 len 32768 data: 14 13 11 13 12 10 12 9 24/30 Read block 555 len 28672 25/30 Write block 395 len 24576 data: 15 15 15 12 9 7 26/30 Read block 144 len 16384 27/30 Write block 122 len 28672 data: 13 13 14 15 13 10 14 28/30 Read block 168 len 8192 29/30 Write block 132 len 32768 data: 13 15 15 13 13 14 13 14 30/30 Write block 511 len 8192 data: 19 19 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 6 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 16 2 20 Done 21 7 21 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1006)) Some(JobId(1016)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:02.909159041Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:02.909377453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1892,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 162 162 162 285 285 285 T F T <--- 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 165 165 165 290 290 290 F F F 6 120-139 164 164 164 289 289 289 T F T <--- 7 140-159 161 161 161 283 283 283 T F T <--- 8 160-179 164 164 164 289 289 289 T F T <--- 9 180-199 166 161 166 292 283 292 F T F <--- 10 200-219 166 162 166 292 285 292 F T F <--- 11 220-239 165 165 165 290 290 290 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 166 165 166 292 290 292 T T T <--- 14 280-299 166 166 166 292 291 292 T T T <--- 15 300-319 166 166 166 292 291 292 F F F <--- 16 320-339 164 164 164 288 288 288 F F F 17 340-359 165 165 165 290 290 290 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 164 164 164 287 287 287 T F T <--- 20 400-419 166 163 166 292 286 292 T T T <--- 21 420-439 166 165 166 292 290 292 F T F <--- 22 440-459 166 162 166 292 285 292 F T F <--- 23 460-479 166 164 166 292 287 292 F T F <--- 24 480-499 152 152 152 271 271 271 F F F 25 500-519 166 166 166 291 291 291 T F T <--- 26 520-539 165 165 165 290 290 290 T F T <--- 27 540-559 166 165 166 292 290 292 F T F <--- 28 560-579 166 166 166 292 291 292 T F T <--- 29 580-599 154 154 154 275 275 275 F F F Max gen: 166, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 283 283 283 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 164 164 164 289 289 289 F F F 5 100-119 165 165 165 290 290 290 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 167 167 167 293 293 293 F F F 8 160-179 167 167 167 293 293 293 F F F 9 180-199 166 166 166 292 292 292 F F F 10 200-219 166 166 166 292 292 292 F F F 11 220-239 165 165 165 290 290 290 F F F 12 240-259 157 157 157 278 278 278 F F F 13 260-279 167 167 167 293 293 293 F F F 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 164 164 164 288 288 288 F F F 17 340-359 165 165 165 290 290 290 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 167 167 167 293 293 293 F F F 20 400-419 167 167 167 293 293 293 F F F 21 420-439 166 166 166 292 292 292 F F F 22 440-459 166 166 166 292 292 292 F F F 23 460-479 166 166 166 292 292 292 F F F 24 480-499 152 152 152 271 271 271 F F F 25 500-519 167 167 167 293 293 293 F F F 26 520-539 167 167 167 293 293 293 F F F 27 540-559 166 166 166 292 292 292 F F F 28 560-579 167 167 167 293 293 293 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 167, Max flush: 293 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.197466099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.205997085Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206331842Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206362808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206371837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206383715Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Crucible b01f0813-0660-4233-8f17-6af452ac23eb has session id: f41fb4ab-b2e1-415e-aff7-84e88536fc06","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206798942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Upstairs opts: Upstairs UUID: b01f0813-0660-4233-8f17-6af452ac23eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206830998Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Crucible stats registered with UUID: b01f0813-0660-4233-8f17-6af452ac23eb","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.206842276Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:17.206852175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207000146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207032451Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20704632Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20720163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207212278Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"b01f0813-0660-4233-8f17-6af452ac23eb active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207236995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207246804Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"0","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207257042Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"1","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207267151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"2","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20727707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207287418Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207297847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"ds_connection connected from Ok(127.0.0.1:43640)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20735332Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"ds_connection connected from Ok(127.0.0.1:55400)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207364698Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"ds_connection connected from Ok(127.0.0.1:36060)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.207380106Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208039661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"1","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208064538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"0","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208164585Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"2","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208499672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[0]R flush_numbers[0..12]: [283, 293, 282, 280, 289, 290, 293, 293, 293, 292, 292, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208522849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[0]R generation[0..12]: [161, 167, 160, 158, 164, 165, 167, 167, 167, 166, 166, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208534457Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208546026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[1]R flush_numbers[0..12]: [283, 293, 282, 280, 289, 290, 293, 293, 293, 292, 292, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208557194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[1]R generation[0..12]: [161, 167, 160, 158, 164, 165, 167, 167, 167, 166, 166, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208568183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208579641Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[2]R flush_numbers[0..12]: [283, 293, 282, 280, 289, 290, 293, 293, 293, 292, 292, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20859067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[2]R generation[0..12]: [161, 167, 160, 158, 164, 165, 167, 167, 167, 166, 166, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208602038Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208613697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208623716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208634094Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208643983Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208655691Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"":"downstairs","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20866588Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208675319Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"0","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208685498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"1","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208695626Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"client":"2","":"downstairs","session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208706025Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208715824Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"b01f0813-0660-4233-8f17-6af452ac23eb is now active with session: f41fb4ab-b2e1-415e-aff7-84e88536fc06","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208726402Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208735581Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"session_id":"f41fb4ab-b2e1-415e-aff7-84e88536fc06"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.20874524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Activated sub_volume b01f0813-0660-4233-8f17-6af452ac23eb","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208753699Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.208762338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.209113462Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"task":"crutest"} 01/30 Write block 158 len 8192 data: 18 19 02/30 Read block 154 len 4096 03/30 Write block 50 len 16384 data: 19 19 18 15 04/30 Read block 178 len 28672 05/30 Read block 241 len 16384 06/30 Read block 126 len 40960 07/30 Write block 5 len 24576 data: 8 11 13 13 14 12 08/30 Read block 162 len 16384 09/30 Write block 210 len 4096 data: 7 10/30 Read block 37 len 12288 11/30 Flush 12/30 Write block 269 len 12288 data: 11 13 17 13/30 Read block 412 len 28672 14/30 Write block 472 len 12288 data: 21 19 13 15/30 Read block 564 len 28672 16/30 Write block 245 len 24576 data: 11 13 13 12 11 13 17/30 Write block 397 len 36864 data: 16 13 10 8 8 11 12 10 15 18/30 Read block 306 len 32768 19/30 Read block 292 len 24576 20/30 Write block 324 len 4096 data: 8 21/30 Read block 231 len 24576 22/30 Read block 230 len 12288 23/30 Flush 24/30 Write block 88 len 28672 data: 11 8 9 9 9 10 11 25/30 Read block 258 len 16384 26/30 Read block 75 len 12288 27/30 Write block 374 len 4096 data: 8 28/30 Read block 182 len 8192 29/30 Write block 485 len 32768 data: 9 9 7 9 11 14 15 14 30/30 Write block 433 len 36864 data: 12 15 14 16 18 17 16 17 17 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Read 7 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 18 22 Done 17 17 1 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:17.218249771Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:17.218497359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1901,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 294 294 294 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 164 164 164 289 289 289 T T F <--- 5 100-119 165 165 165 290 290 290 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 168 168 168 294 294 294 F F F 8 160-179 167 167 167 293 293 293 F F F 9 180-199 166 166 166 292 292 292 F F F 10 200-219 168 168 168 294 294 294 F F F 11 220-239 165 165 165 290 290 290 F F F 12 240-259 168 168 157 295 295 278 F F F <--- 13 260-279 168 168 167 295 295 293 F F T <--- 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 168 168 164 295 295 288 F F F <--- 17 340-359 165 165 165 290 290 290 F F F 18 360-379 164 164 164 287 287 287 T T F <--- 19 380-399 168 168 167 295 295 293 F F F <--- 20 400-419 168 168 167 295 295 293 F F F <--- 21 420-439 166 166 166 292 292 292 T T F <--- 22 440-459 166 166 166 292 292 292 T T F <--- 23 460-479 168 168 166 295 295 292 F F F <--- 24 480-499 152 152 152 271 271 271 T T F <--- 25 500-519 167 167 167 293 293 293 F F F 26 520-539 167 167 167 293 293 293 F F F 27 540-559 166 166 166 292 292 292 F F F 28 560-579 167 167 167 293 293 293 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 168, Max flush: 295 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 294 294 294 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 169 169 169 296 296 296 F F F 5 100-119 165 165 165 290 290 290 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 168 168 168 294 294 294 F F F 8 160-179 167 167 167 293 293 293 F F F 9 180-199 166 166 166 292 292 292 F F F 10 200-219 168 168 168 294 294 294 F F F 11 220-239 165 165 165 290 290 290 F F F 12 240-259 168 168 168 295 295 295 F F F 13 260-279 168 168 168 295 295 295 F F F 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 168 168 168 295 295 295 F F F 17 340-359 165 165 165 290 290 290 F F F 18 360-379 169 169 169 296 296 296 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 169 169 169 296 296 296 F F F 22 440-459 169 169 169 296 296 296 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 169 169 169 296 296 296 F F F 25 500-519 167 167 167 293 293 293 F F F 26 520-539 167 167 167 293 293 293 F F F 27 540-559 166 166 166 292 292 292 F F F 28 560-579 167 167 167 293 293 293 F F F 29 580-599 154 154 154 275 275 275 F F F Max gen: 169, Max flush: 296 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.483189351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.492328948Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.492679263Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.492718118Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.492727617Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.492740325Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Crucible f1a30052-d360-4a1a-9ccf-02604fd9aaf3 has session id: 6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493137664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Upstairs opts: Upstairs UUID: f1a30052-d360-4a1a-9ccf-02604fd9aaf3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493162461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Crucible stats registered with UUID: f1a30052-d360-4a1a-9ccf-02604fd9aaf3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493173539Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:31.493183648Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493350146Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493384472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.49339669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493547611Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493566868Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"f1a30052-d360-4a1a-9ccf-02604fd9aaf3 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493597794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493608313Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"0","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493619502Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"1","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.49363049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"2","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493641279Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493652627Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493701521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62551)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493752514Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35509)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493779401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62847)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.493792229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494477031Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"0","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494513606Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"1","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494527044Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"2","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494881818Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[0]R flush_numbers[0..12]: [294, 293, 294, 280, 296, 290, 293, 294, 293, 292, 294, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494914814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[0]R generation[0..12]: [168, 167, 168, 158, 169, 165, 167, 168, 167, 166, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494927512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494940051Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[1]R flush_numbers[0..12]: [294, 293, 294, 280, 296, 290, 293, 294, 293, 292, 294, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494952099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[1]R generation[0..12]: [168, 167, 168, 158, 169, 165, 167, 168, 167, 166, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494964098Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494976496Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[2]R flush_numbers[0..12]: [294, 293, 294, 280, 296, 290, 293, 294, 293, 292, 294, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.494989654Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[2]R generation[0..12]: [168, 167, 168, 158, 169, 165, 167, 168, 167, 166, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495001463Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495013821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.49502442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495035678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495046367Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495057016Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"":"downstairs","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495067974Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495077713Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"0","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495088672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"1","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.49509946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"client":"2","":"downstairs","session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495110609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495121087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"f1a30052-d360-4a1a-9ccf-02604fd9aaf3 is now active with session: 6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495132356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495142565Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495152763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"session_id":"6ee2dc1c-9ab0-487b-a601-1aab9e1bfd8a"} {"msg":"Activated sub_volume f1a30052-d360-4a1a-9ccf-02604fd9aaf3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495163192Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495172351Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.495550762Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"task":"crutest"} 01/30 Write block 188 len 4096 data: 12 02/30 Read block 176 len 36864 03/30 Read block 578 len 32768 04/30 Read block 153 len 4096 05/30 Write block 200 len 16384 data: 13 12 15 12 06/30 Read block 188 len 40960 07/30 Read block 5 len 32768 08/30 Write block 429 len 40960 data: 8 8 11 11 13 16 15 17 19 18 09/30 Write block 100 len 28672 data: 15 13 12 14 13 16 18 10/30 Write block 586 len 40960 data: 13 12 12 12 13 11 14 15 12 12 11/30 Write block 452 len 12288 data: 10 9 10 12/30 Read block 461 len 28672 13/30 Write block 524 len 4096 data: 15 14/30 Write block 49 len 12288 data: 18 20 20 15/30 Read block 373 len 20480 16/30 Read block 265 len 16384 17/30 Write block 550 len 12288 data: 10 9 9 18/30 Read block 510 len 32768 19/30 Write block 267 len 24576 data: 11 12 12 14 18 17 20/30 Read block 74 len 12288 21/30 Write block 503 len 32768 data: 13 15 12 15 15 15 13 13 22/30 Write block 175 len 40960 data: 15 15 18 16 11 11 12 13 18 16 23/30 Flush 24/30 Write block 152 len 12288 data: 12 11 13 25/30 Write block 9 len 16384 data: 15 13 16 12 26/30 Write block 503 len 4096 data: 14 27/30 Read block 29 len 28672 28/30 Write block 448 len 32768 data: 11 12 11 9 11 10 11 7 29/30 Write block 184 len 24576 data: 17 15 17 13 13 11 30/30 Read block 352 len 16384 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 8 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:31.50437656Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:31.504604141Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1910,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 294 294 294 T T F <--- 1 020-039 167 167 167 293 293 293 F F F 2 040-059 170 170 168 297 297 294 F F F <--- 3 060-079 158 158 158 280 280 280 F F F 4 080-099 169 169 169 296 296 296 F F F 5 100-119 170 170 165 297 297 290 F F F <--- 6 120-139 167 167 167 293 293 293 F F F 7 140-159 168 168 168 294 294 294 T T F <--- 8 160-179 170 170 167 297 297 293 F F F <--- 9 180-199 170 170 166 297 297 292 T T T <--- 10 200-219 170 170 168 297 297 294 F F F <--- 11 220-239 165 165 165 290 290 290 F F F 12 240-259 168 168 168 295 295 295 F F F 13 260-279 170 170 168 297 297 295 F F F <--- 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 168 168 168 295 295 295 F F F 17 340-359 165 165 165 290 290 290 F F F 18 360-379 169 169 169 296 296 296 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 170 170 169 297 297 296 F F F <--- 22 440-459 170 170 169 297 297 296 T T F <--- 23 460-479 168 168 168 295 295 295 F F F 24 480-499 169 169 169 296 296 296 F F F 25 500-519 170 170 167 297 297 293 T T F <--- 26 520-539 170 170 167 297 297 293 F F F <--- 27 540-559 170 170 166 297 297 292 F F F <--- 28 560-579 167 167 167 293 293 293 F F F 29 580-599 170 170 154 297 297 275 F F F <--- Max gen: 170, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 298 298 298 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 170 170 170 297 297 297 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 169 169 169 296 296 296 F F F 5 100-119 170 170 170 297 297 297 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 171 171 171 298 298 298 F F F 8 160-179 170 170 170 297 297 297 F F F 9 180-199 171 171 171 298 298 298 F F F 10 200-219 170 170 170 297 297 297 F F F 11 220-239 165 165 165 290 290 290 F F F 12 240-259 168 168 168 295 295 295 F F F 13 260-279 170 170 170 297 297 297 F F F 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 168 168 168 295 295 295 F F F 17 340-359 165 165 165 290 290 290 F F F 18 360-379 169 169 169 296 296 296 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 298 298 298 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 169 169 169 296 296 296 F F F 25 500-519 171 171 171 298 298 298 F F F 26 520-539 170 170 170 297 297 297 F F F 27 540-559 170 170 170 297 297 297 F F F 28 560-579 167 167 167 293 293 293 F F F 29 580-599 170 170 170 297 297 297 F F F Max gen: 171, Max flush: 298 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.782146411Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.790835767Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791143177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791175393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791184282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.79119656Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Crucible 654cf834-6e78-48df-ac82-8cc250ca867d has session id: 329e5130-7fe6-4f19-a14c-82160b0fbb11","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791515229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Upstairs opts: Upstairs UUID: 654cf834-6e78-48df-ac82-8cc250ca867d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791534856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Crucible stats registered with UUID: 654cf834-6e78-48df-ac82-8cc250ca867d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791560863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:45.791603478Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791754728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791787004Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791798972Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791938364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791956262Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"654cf834-6e78-48df-ac82-8cc250ca867d active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791983748Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.791993747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"0","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792014464Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"1","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792024803Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"2","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792034922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.79204554Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792056309Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"ds_connection connected from Ok(127.0.0.1:64027)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792108662Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"ds_connection connected from Ok(127.0.0.1:55838)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792120461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"ds_connection connected from Ok(127.0.0.1:45344)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792139408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792881222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"1","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792908979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"2","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.792921247Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"0","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793353651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[0]R flush_numbers[0..12]: [298, 293, 297, 280, 296, 297, 293, 298, 297, 298, 297, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793378538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[0]R generation[0..12]: [171, 167, 170, 158, 169, 170, 167, 171, 170, 171, 170, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793391296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793403525Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[1]R flush_numbers[0..12]: [298, 293, 297, 280, 296, 297, 293, 298, 297, 298, 297, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793414813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[1]R generation[0..12]: [171, 167, 170, 158, 169, 170, 167, 171, 170, 171, 170, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793426002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.79343748Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[2]R flush_numbers[0..12]: [298, 293, 297, 280, 296, 297, 293, 298, 297, 298, 297, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793449009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[2]R generation[0..12]: [171, 167, 170, 158, 169, 170, 167, 171, 170, 171, 170, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793460357Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793471956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793482085Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793492543Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793504442Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.79352036Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"":"downstairs","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793530598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793539937Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"0","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793550446Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"1","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793560984Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"client":"2","":"downstairs","session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793571573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793581402Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"654cf834-6e78-48df-ac82-8cc250ca867d is now active with session: 329e5130-7fe6-4f19-a14c-82160b0fbb11","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.79359187Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793601299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"session_id":"329e5130-7fe6-4f19-a14c-82160b0fbb11"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793610948Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Activated sub_volume 654cf834-6e78-48df-ac82-8cc250ca867d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793619407Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793628136Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.793987849Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"task":"crutest"} 01/30 Write block 369 len 36864 data: 17 15 11 9 8 9 6 7 9 02/30 Write block 493 len 36864 data: 15 13 14 12 11 10 10 10 12 03/30 Read block 97 len 24576 04/30 Read block 530 len 28672 05/30 Write block 11 len 4096 data: 17 06/30 Read block 27 len 32768 07/30 Read block 67 len 32768 08/30 Read block 150 len 32768 09/30 Read block 106 len 36864 10/30 Read block 4 len 32768 11/30 Read block 595 len 4096 12/30 Read block 200 len 20480 13/30 Read block 345 len 8192 14/30 Write block 254 len 40960 data: 9 6 8 7 6 7 9 7 9 9 15/30 Flush 16/30 Write block 270 len 16384 data: 15 19 18 18 17/30 Write block 489 len 20480 data: 12 15 16 15 16 18/30 Flush 19/30 Write block 85 len 4096 data: 9 20/30 Write block 342 len 16384 data: 9 9 10 13 21/30 Write block 576 len 36864 data: 17 16 15 13 15 12 11 12 11 22/30 Read block 580 len 4096 23/30 Read block 502 len 12288 24/30 Read block 570 len 24576 25/30 Write block 328 len 36864 data: 16 15 11 13 14 14 12 11 8 26/30 Write block 113 len 4096 data: 12 27/30 Write block 81 len 24576 data: 9 11 6 9 10 10 28/30 Read block 98 len 40960 29/30 Write block 202 len 16384 data: 16 13 14 12 30/30 Write block 520 len 20480 data: 19 17 19 17 16 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:06:45.802398571Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:06:45.802601785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1919,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 171 299 299 298 F F F <--- 1 020-039 167 167 167 293 293 293 F F F 2 040-059 170 170 170 297 297 297 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 169 169 169 296 296 296 T T F <--- 5 100-119 170 170 170 297 297 297 T T F <--- 6 120-139 167 167 167 293 293 293 F F F 7 140-159 171 171 171 298 298 298 F F F 8 160-179 170 170 170 297 297 297 F F F 9 180-199 171 171 171 298 298 298 F F F 10 200-219 170 170 170 297 297 297 T T F <--- 11 220-239 165 165 165 290 290 290 F F F 12 240-259 172 172 168 299 299 295 F F F <--- 13 260-279 172 172 170 300 300 297 F F F <--- 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 168 168 168 295 295 295 T T F <--- 17 340-359 165 165 165 290 290 290 T T F <--- 18 360-379 172 172 169 299 299 296 F F F <--- 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 298 298 298 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 172 172 169 300 300 296 F F F <--- 25 500-519 172 172 171 299 299 298 F F F <--- 26 520-539 170 170 170 297 297 297 T T F <--- 27 540-559 170 170 170 297 297 297 F F F 28 560-579 167 167 167 293 293 293 T T F <--- 29 580-599 170 170 170 297 297 297 T T F <--- Max gen: 172, Max flush: 300 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 299 299 299 F F F 1 020-039 167 167 167 293 293 293 F F F 2 040-059 170 170 170 297 297 297 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 173 173 173 301 301 301 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 171 171 171 298 298 298 F F F 8 160-179 170 170 170 297 297 297 F F F 9 180-199 171 171 171 298 298 298 F F F 10 200-219 173 173 173 301 301 301 F F F 11 220-239 165 165 165 290 290 290 F F F 12 240-259 172 172 172 299 299 299 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 167 167 167 293 293 293 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 173 173 173 301 301 301 F F F 18 360-379 172 172 172 299 299 299 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 298 298 298 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 172 172 172 300 300 300 F F F 25 500-519 172 172 172 299 299 299 F F F 26 520-539 173 173 173 301 301 301 F F F 27 540-559 170 170 170 297 297 297 F F F 28 560-579 173 173 173 301 301 301 F F F 29 580-599 173 173 173 301 301 301 F F F Max gen: 173, Max flush: 301 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.086697627Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09556501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.095887858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.095929503Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.095939301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09595209Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Crucible a5add3a4-9396-442d-a5c3-80b274511f26 has session id: 7f5cb7f5-6368-4233-a6ab-3be321af93e9","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096348689Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Upstairs opts: Upstairs UUID: a5add3a4-9396-442d-a5c3-80b274511f26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096381814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Crucible stats registered with UUID: a5add3a4-9396-442d-a5c3-80b274511f26","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096397522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:00.096408061Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096521876Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096541224Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096552552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096750886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096764515Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"a5add3a4-9396-442d-a5c3-80b274511f26 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096809119Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096819838Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"0","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096832686Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"1","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096843784Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"2","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096854723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096866052Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09687724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56236)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09695841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61506)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.096987886Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51589)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.097046608Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.097766625Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"0","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.097800601Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"1","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.097837326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"2","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098148846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[0]R flush_numbers[0..12]: [299, 293, 297, 280, 301, 301, 293, 298, 297, 298, 301, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098181891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[0]R generation[0..12]: [172, 167, 170, 158, 173, 173, 167, 171, 170, 171, 173, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09819461Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098209538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[1]R flush_numbers[0..12]: [299, 293, 297, 280, 301, 301, 293, 298, 297, 298, 301, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098221606Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[1]R generation[0..12]: [172, 167, 170, 158, 173, 173, 167, 171, 170, 171, 173, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098233485Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098245803Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[2]R flush_numbers[0..12]: [299, 293, 297, 280, 301, 301, 293, 298, 297, 298, 301, 290]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098257832Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[2]R generation[0..12]: [172, 167, 170, 158, 173, 173, 167, 171, 170, 171, 173, 165]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09826967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098282008Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098292587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098303666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098314064Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098324623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"":"downstairs","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098335452Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09834525Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"0","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098356009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"1","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098367137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"client":"2","":"downstairs","session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098379706Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098389875Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"a5add3a4-9396-442d-a5c3-80b274511f26 is now active with session: 7f5cb7f5-6368-4233-a6ab-3be321af93e9","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098401113Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098410922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"session_id":"7f5cb7f5-6368-4233-a6ab-3be321af93e9"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.09842109Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Activated sub_volume a5add3a4-9396-442d-a5c3-80b274511f26","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098430379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098439728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.098795032Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"task":"crutest"} 01/30 Write block 213 len 40960 data: 8 7 11 14 17 20 21 19 19 20 02/30 Write block 430 len 32768 data: 9 12 12 14 17 16 18 20 03/30 Write block 550 len 4096 data: 11 04/30 Flush 05/30 Write block 508 len 32768 data: 16 14 14 20 20 16 14 10 06/30 Write block 15 len 24576 data: 13 13 14 14 15 15 07/30 Write block 240 len 36864 data: 10 11 13 13 14 12 14 14 13 08/30 Flush 09/30 Write block 247 len 12288 data: 15 14 12 10/30 Write block 220 len 4096 data: 20 11/30 Write block 509 len 8192 data: 15 15 12/30 Read block 431 len 12288 13/30 Read block 226 len 28672 14/30 Flush 15/30 Write block 361 len 28672 data: 21 26 24 26 23 22 21 16/30 Read block 438 len 12288 17/30 Write block 99 len 4096 data: 18 18/30 Flush 19/30 Write block 285 len 8192 data: 13 14 20/30 Flush 21/30 Write block 251 len 16384 data: 12 11 14 10 22/30 Flush 23/30 Write block 477 len 24576 data: 10 10 9 9 9 11 24/30 Write block 142 len 28672 data: 11 10 14 13 12 10 12 25/30 Read block 581 len 40960 26/30 Read block 481 len 16384 27/30 Read block 560 len 4096 28/30 Read block 147 len 12288 29/30 Write block 441 len 12288 data: 18 18 17 30/30 Read block 480 len 24576 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:00.108157811Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:00.108379072Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1928,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 172 174 303 299 303 F F F <--- 1 020-039 174 167 174 303 293 303 F F F <--- 2 040-059 170 170 170 297 297 297 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 174 173 174 305 301 305 F F F <--- 5 100-119 173 173 173 301 301 301 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 171 171 171 298 298 298 T F T <--- 8 160-179 170 170 170 297 297 297 F F F 9 180-199 171 171 171 298 298 298 F F F 10 200-219 174 173 174 302 301 302 F F F <--- 11 220-239 174 165 174 304 290 304 F F F <--- 12 240-259 174 172 174 307 299 307 F F F <--- 13 260-279 172 172 172 300 300 300 F F F 14 280-299 174 167 174 306 293 306 F F F <--- 15 300-319 166 166 166 292 292 292 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 173 173 173 301 301 301 F F F 18 360-379 174 172 174 305 299 305 F F F <--- 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 174 170 174 302 297 302 F F F <--- 22 440-459 171 171 171 298 298 298 T F T <--- 23 460-479 168 168 168 295 295 295 T F T <--- 24 480-499 172 172 172 300 300 300 T F T <--- 25 500-519 174 172 174 304 299 304 F F F <--- 26 520-539 173 173 173 301 301 301 F F F 27 540-559 174 170 174 302 297 302 F F F <--- 28 560-579 173 173 173 301 301 301 F F F 29 580-599 173 173 173 301 301 301 F F F Max gen: 174, Max flush: 307 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 303 303 303 F F F 1 020-039 174 174 174 303 303 303 F F F 2 040-059 170 170 170 297 297 297 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 174 174 174 305 305 305 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 167 167 167 293 293 293 F F F 7 140-159 175 175 175 308 308 308 F F F 8 160-179 170 170 170 297 297 297 F F F 9 180-199 171 171 171 298 298 298 F F F 10 200-219 174 174 174 302 302 302 F F F 11 220-239 174 174 174 304 304 304 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 174 174 174 306 306 306 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 173 173 173 301 301 301 F F F 18 360-379 174 174 174 305 305 305 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 174 174 174 302 302 302 F F F 22 440-459 175 175 175 308 308 308 F F F 23 460-479 175 175 175 308 308 308 F F F 24 480-499 175 175 175 308 308 308 F F F 25 500-519 174 174 174 304 304 304 F F F 26 520-539 173 173 173 301 301 301 F F F 27 540-559 174 174 174 302 302 302 F F F 28 560-579 173 173 173 301 301 301 F F F 29 580-599 173 173 173 301 301 301 F F F Max gen: 175, Max flush: 308 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.394564185Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.403248151Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.403561551Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.403592127Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.403601726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.403613924Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Crucible 76d971b6-a403-4264-bb7c-9a109b7735d5 has session id: 105632c1-5ed1-4bc1-ad72-b7606e8d36f6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404005084Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Upstairs opts: Upstairs UUID: 76d971b6-a403-4264-bb7c-9a109b7735d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404051977Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Crucible stats registered with UUID: 76d971b6-a403-4264-bb7c-9a109b7735d5","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404066866Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:14.404076794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404219596Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404254711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404275808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404524926Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404535925Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"76d971b6-a403-4264-bb7c-9a109b7735d5 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404554952Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404564441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"0","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.40457507Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"1","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404585758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"2","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404596147Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404607046Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.40464882Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63076)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404660669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53506)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404692795Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58376)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.404709242Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405392524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"0","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405406152Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"1","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405543455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"2","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405831667Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[0]R flush_numbers[0..12]: [303, 303, 297, 280, 305, 301, 293, 308, 297, 298, 302, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405868303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[0]R generation[0..12]: [174, 174, 170, 158, 174, 173, 167, 175, 170, 171, 174, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405880421Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405892419Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[1]R flush_numbers[0..12]: [303, 303, 297, 280, 305, 301, 293, 308, 297, 298, 302, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405903978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[1]R generation[0..12]: [174, 174, 170, 158, 174, 173, 167, 175, 170, 171, 174, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405915396Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405927275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[2]R flush_numbers[0..12]: [303, 303, 297, 280, 305, 301, 293, 308, 297, 298, 302, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405939023Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[2]R generation[0..12]: [174, 174, 170, 158, 174, 173, 167, 175, 170, 171, 174, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405951962Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.40596373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405974069Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405984837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.405994736Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406004885Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"":"downstairs","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406015453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406025022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"0","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406035281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"1","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.40604564Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"client":"2","":"downstairs","session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406056198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406066007Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"76d971b6-a403-4264-bb7c-9a109b7735d5 is now active with session: 105632c1-5ed1-4bc1-ad72-b7606e8d36f6","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406076946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406086564Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"session_id":"105632c1-5ed1-4bc1-ad72-b7606e8d36f6"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406096373Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Activated sub_volume 76d971b6-a403-4264-bb7c-9a109b7735d5","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406105082Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406113891Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.406470605Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"task":"crutest"} 01/30 Write block 361 len 28672 data: 22 27 25 27 24 23 22 02/30 Write block 150 len 8192 data: 12 10 03/30 Write block 180 len 4096 data: 12 04/30 Write block 414 len 12288 data: 8 8 11 05/30 Write block 546 len 20480 data: 12 12 11 11 12 06/30 Read block 468 len 28672 07/30 Read block 206 len 4096 08/30 Read block 212 len 36864 09/30 Read block 518 len 24576 10/30 Write block 161 len 40960 data: 17 16 13 12 11 11 13 15 14 13 11/30 Read block 394 len 24576 12/30 Read block 493 len 28672 13/30 Read block 226 len 20480 14/30 Write block 120 len 40960 data: 11 14 14 14 15 16 14 11 15 15 15/30 Write block 355 len 36864 data: 18 15 14 18 20 24 23 28 26 16/30 Read block 60 len 4096 17/30 Write block 202 len 24576 data: 17 14 15 13 10 14 18/30 Write block 435 len 40960 data: 17 19 21 19 17 18 19 19 18 15 19/30 Read block 450 len 24576 20/30 Read block 585 len 12288 21/30 Read block 331 len 4096 22/30 Read block 226 len 20480 23/30 Write block 32 len 40960 data: 10 10 10 10 12 9 10 10 13 13 24/30 Read block 175 len 28672 25/30 Read block 590 len 12288 26/30 Read block 307 len 4096 27/30 Read block 330 len 8192 28/30 Write block 505 len 28672 data: 13 16 16 17 16 16 21 29/30 Write block 554 len 36864 data: 10 10 11 9 11 13 15 17 17 30/30 Read block 79 len 28672 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:14.41570636Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:14.415912773Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1937,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 303 303 303 F F F 1 020-039 174 174 174 303 303 303 T F T <--- 2 040-059 170 170 170 297 297 297 T F T <--- 3 060-079 158 158 158 280 280 280 F F F 4 080-099 174 174 174 305 305 305 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 167 167 167 293 293 293 T F T <--- 7 140-159 175 175 175 308 308 308 T T T 8 160-179 170 170 170 297 297 297 T F T <--- 9 180-199 171 171 171 298 298 298 T T T 10 200-219 174 174 174 302 302 302 T F T <--- 11 220-239 174 174 174 304 304 304 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 174 174 174 306 306 306 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 173 173 173 301 301 301 T F T <--- 18 360-379 174 174 174 305 305 305 T T T 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 T T T 21 420-439 174 174 174 302 302 302 T F T <--- 22 440-459 175 175 175 308 308 308 T F T <--- 23 460-479 175 175 175 308 308 308 F F F 24 480-499 175 175 175 308 308 308 F F F 25 500-519 174 174 174 304 304 304 T F T <--- 26 520-539 173 173 173 301 301 301 F F F 27 540-559 174 174 174 302 302 302 T F T <--- 28 560-579 173 173 173 301 301 301 T F T <--- 29 580-599 173 173 173 301 301 301 F F F Max gen: 175, Max flush: 308 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 303 303 303 F F F 1 020-039 176 176 176 309 309 309 F F F 2 040-059 176 176 176 309 309 309 F F F 3 060-079 158 158 158 280 280 280 F F F 4 080-099 174 174 174 305 305 305 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 176 176 176 309 309 309 F F F 7 140-159 176 176 176 309 309 309 F F F 8 160-179 176 176 176 309 309 309 F F F 9 180-199 176 176 176 309 309 309 F F F 10 200-219 176 176 176 309 309 309 F F F 11 220-239 174 174 174 304 304 304 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 174 174 174 306 306 306 F F F 15 300-319 166 166 166 292 292 292 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 176 176 176 309 309 309 F F F 18 360-379 176 176 176 309 309 309 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 176 176 176 309 309 309 F F F 21 420-439 176 176 176 309 309 309 F F F 22 440-459 176 176 176 309 309 309 F F F 23 460-479 175 175 175 308 308 308 F F F 24 480-499 175 175 175 308 308 308 F F F 25 500-519 176 176 176 309 309 309 F F F 26 520-539 173 173 173 301 301 301 F F F 27 540-559 176 176 176 309 309 309 F F F 28 560-579 176 176 176 309 309 309 F F F 29 580-599 173 173 173 301 301 301 F F F Max gen: 176, Max flush: 309 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.699646703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:28.707151682Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.707209024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.714559354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.714867364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.71489863Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.714907839Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.714920197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Crucible a3892228-f71b-4470-8e65-78e531093417 has session id: 0bd77161-85d4-43b5-a5aa-73ad352e253d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715345472Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Upstairs opts: Upstairs UUID: a3892228-f71b-4470-8e65-78e531093417, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715364929Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Crucible stats registered with UUID: a3892228-f71b-4470-8e65-78e531093417","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715378758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:28.715388266Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715531488Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715549796Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715560364Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715717734Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715730862Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"a3892228-f71b-4470-8e65-78e531093417 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715766318Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715776326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"0","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715786625Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"1","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715796904Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"2","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715806922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715817681Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715833019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63177)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715920408Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55887)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.715996288Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33042)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.716009816Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.716606329Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"1","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.7166753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"2","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.716688028Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"0","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717020145Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[0]R flush_numbers[0..12]: [303, 309, 309, 280, 305, 301, 309, 309, 309, 309, 309, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717047632Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[0]R generation[0..12]: [174, 176, 176, 158, 174, 173, 176, 176, 176, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.71705988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717071989Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[1]R flush_numbers[0..12]: [303, 309, 309, 280, 305, 301, 309, 309, 309, 309, 309, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717083387Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[1]R generation[0..12]: [174, 176, 176, 158, 174, 173, 176, 176, 176, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717094666Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717106354Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[2]R flush_numbers[0..12]: [303, 309, 309, 280, 305, 301, 309, 309, 309, 309, 309, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717117753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[2]R generation[0..12]: [174, 176, 176, 158, 174, 173, 176, 176, 176, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717129131Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.71714094Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Max found gen is 177","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717151078Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Generation requested: 178 >= found:177","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717161697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717173246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717183394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"":"downstairs","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717193993Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717203322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"0","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.71721372Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"1","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717224049Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"client":"2","":"downstairs","session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717234558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717244296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"a3892228-f71b-4470-8e65-78e531093417 is now active with session: 0bd77161-85d4-43b5-a5aa-73ad352e253d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717254995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717264484Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"session_id":"0bd77161-85d4-43b5-a5aa-73ad352e253d"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717274343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Activated sub_volume a3892228-f71b-4470-8e65-78e531093417","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717283131Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717292Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.717642745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} 01/30 Write block 185 len 24576 data: 16 18 14 14 12 11 02/30 Write block 406 len 8192 data: 14 15 03/30 Read block 89 len 8192 04/30 Read block 218 len 8192 05/30 Write block 7 len 32768 data: 14 14 16 14 18 13 11 11 06/30 Write block 511 len 36864 data: 22 21 17 15 11 11 16 13 16 07/30 Write block 268 len 16384 data: 13 13 16 20 08/30 Write block 589 len 24576 data: 13 14 12 15 16 13 09/30 Write block 129 len 24576 data: 16 15 13 14 16 16 10/30 Read block 236 len 20480 11/30 Read block 277 len 32768 12/30 Read block 125 len 40960 13/30 Read block 474 len 8192 14/30 Read block 49 len 16384 15/30 Read block 49 len 36864 16/30 Write block 528 len 12288 data: 19 16 15 17/30 Write block 78 len 32768 data: 10 11 10 10 12 7 10 11 18/30 Write block 518 len 4096 data: 14 19/30 Flush 20/30 Write block 309 len 40960 data: 17 16 19 16 15 15 14 13 14 9 21/30 Read block 251 len 12288 22/30 Read block 383 len 40960 23/30 Write block 171 len 40960 data: 16 13 14 13 16 16 19 17 12 13 24/30 Read block 440 len 24576 25/30 Flush 26/30 Write block 189 len 40960 data: 13 12 14 15 11 6 8 9 8 10 27/30 Read block 467 len 36864 28/30 Write block 549 len 12288 data: 12 13 10 29/30 Write block 424 len 12288 data: 8 7 7 30/30 Write block 177 len 8192 data: 20 18 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 3 Sent Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 2 3 28 Done 7 28 27 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:28.726769024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:28.726974068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1946,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 178 178 303 310 310 T F F <--- 1 020-039 176 176 176 309 309 309 F F F 2 040-059 176 176 176 309 309 309 F F F 3 060-079 158 178 178 280 310 310 F F F <--- 4 080-099 174 178 178 305 310 310 F F F <--- 5 100-119 173 173 173 301 301 301 F F F 6 120-139 176 178 178 309 310 310 F F F <--- 7 140-159 176 176 176 309 309 309 F F F 8 160-179 176 178 178 309 311 311 F T T <--- 9 180-199 176 178 178 309 311 311 T T T <--- 10 200-219 176 176 176 309 309 309 F F F 11 220-239 174 174 174 304 304 304 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 172 178 178 300 310 310 T F F <--- 14 280-299 174 174 174 306 306 306 F F F 15 300-319 166 178 178 292 311 311 F F F <--- 16 320-339 173 173 173 301 301 301 F F F 17 340-359 176 176 176 309 309 309 F F F 18 360-379 176 176 176 309 309 309 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 176 178 178 309 310 310 T F F <--- 21 420-439 176 176 176 309 309 309 F T T <--- 22 440-459 176 176 176 309 309 309 F F F 23 460-479 175 175 175 308 308 308 F F F 24 480-499 175 175 175 308 308 308 F F F 25 500-519 176 178 178 309 310 310 T F F <--- 26 520-539 173 178 178 301 310 310 F F F <--- 27 540-559 176 176 176 309 309 309 F T T <--- 28 560-579 176 176 176 309 309 309 F F F 29 580-599 173 178 178 301 310 310 F F F <--- Max gen: 178, Max flush: 311 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 310 310 310 F F F 1 020-039 176 176 176 309 309 309 F F F 2 040-059 176 176 176 309 309 309 F F F 3 060-079 178 178 178 310 310 310 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 176 176 176 309 309 309 F F F 8 160-179 179 179 179 312 312 312 F F F 9 180-199 179 179 179 312 312 312 F F F 10 200-219 176 176 176 309 309 309 F F F 11 220-239 174 174 174 304 304 304 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 178 178 178 310 310 310 F F F 14 280-299 174 174 174 306 306 306 F F F 15 300-319 178 178 178 311 311 311 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 176 176 176 309 309 309 F F F 18 360-379 176 176 176 309 309 309 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 178 178 178 310 310 310 F F F 21 420-439 179 179 179 312 312 312 F F F 22 440-459 176 176 176 309 309 309 F F F 23 460-479 175 175 175 308 308 308 F F F 24 480-499 175 175 175 308 308 308 F F F 25 500-519 178 178 178 310 310 310 F F F 26 520-539 178 178 178 310 310 310 F F F 27 540-559 179 179 179 312 312 312 F F F 28 560-579 176 176 176 309 309 309 F F F 29 580-599 178 178 178 310 310 310 F F F Max gen: 179, Max flush: 312 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.006825509Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.01608966Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.016431956Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.016473211Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.01648341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.016496558Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Crucible 6fecc463-ef4e-4a93-8667-9d1ea439082d has session id: 62e19d10-b8a3-4bcd-930b-67f7b5e33938","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.016908325Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Upstairs opts: Upstairs UUID: 6fecc463-ef4e-4a93-8667-9d1ea439082d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.0169417Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Crucible stats registered with UUID: 6fecc463-ef4e-4a93-8667-9d1ea439082d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.016957498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:43.016968067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017114928Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017133755Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017145214Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017312002Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017323071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"6fecc463-ef4e-4a93-8667-9d1ea439082d active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017342178Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017354227Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"0","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017365285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"1","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017376024Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"2","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017386733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017397881Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.01740908Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"ds_connection connected from Ok(127.0.0.1:59381)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017498198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"ds_connection connected from Ok(127.0.0.1:49175)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017510817Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"ds_connection connected from Ok(127.0.0.1:35725)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.017609234Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018322912Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"0","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018346329Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"2","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018451835Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"1","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018748787Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[0]R flush_numbers[0..12]: [310, 309, 309, 310, 310, 301, 310, 309, 312, 312, 309, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018778253Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[0]R generation[0..12]: [178, 176, 176, 178, 178, 173, 178, 176, 179, 179, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018791041Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018806079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[1]R flush_numbers[0..12]: [310, 309, 309, 310, 310, 301, 310, 309, 312, 312, 309, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018818198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[1]R generation[0..12]: [178, 176, 176, 178, 178, 173, 178, 176, 179, 179, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018830256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018842574Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[2]R flush_numbers[0..12]: [310, 309, 309, 310, 310, 301, 310, 309, 312, 312, 309, 304]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018854583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[2]R generation[0..12]: [178, 176, 176, 178, 178, 173, 178, 176, 179, 179, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018866411Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.0188786Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018889578Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018900587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018911066Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018921744Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"":"downstairs","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018932683Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018942511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"0","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.01895328Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"1","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018964609Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"client":"2","":"downstairs","session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018977087Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018987396Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"6fecc463-ef4e-4a93-8667-9d1ea439082d is now active with session: 62e19d10-b8a3-4bcd-930b-67f7b5e33938","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.018998734Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.019008633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"session_id":"62e19d10-b8a3-4bcd-930b-67f7b5e33938"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.019018932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Activated sub_volume 6fecc463-ef4e-4a93-8667-9d1ea439082d","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.01902798Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.019037189Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.019397733Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"task":"crutest"} 01/30 Write block 151 len 28672 data: 11 13 12 14 13 16 16 02/30 Read block 403 len 24576 03/30 Write block 221 len 32768 data: 20 21 17 17 17 16 17 14 04/30 Flush 05/30 Write block 557 len 28672 data: 10 12 14 16 18 18 14 06/30 Write block 539 len 32768 data: 18 21 20 20 16 13 13 13 07/30 Write block 456 len 24576 data: 7 10 12 13 12 13 08/30 Flush 09/30 Write block 429 len 36864 data: 9 10 13 13 15 18 18 20 22 10/30 Flush 11/30 Write block 472 len 12288 data: 22 20 14 12/30 Write block 33 len 4096 data: 11 13/30 Write block 160 len 28672 data: 18 18 17 14 13 12 12 14/30 Write block 579 len 40960 data: 14 16 13 12 13 12 11 14 13 13 15/30 Read block 219 len 16384 16/30 Write block 458 len 12288 data: 13 14 13 17/30 Write block 488 len 24576 data: 10 13 16 17 16 17 18/30 Write block 62 len 36864 data: 14 13 11 11 12 13 12 10 8 19/30 Read block 34 len 8192 20/30 Read block 403 len 12288 21/30 Write block 496 len 12288 data: 13 12 11 22/30 Write block 47 len 32768 data: 21 20 19 21 21 19 16 14 23/30 Read block 23 len 4096 24/30 Read block 132 len 20480 25/30 Write block 290 len 8192 data: 13 13 26/30 Read block 157 len 24576 27/30 Write block 176 len 20480 data: 17 21 19 13 14 28/30 Write block 149 len 24576 data: 13 13 12 14 13 15 29/30 Write block 40 len 20480 data: 14 14 15 15 17 30/30 Write block 350 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 3 31 Done 24 0 23 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1003)) Some(JobId(1009)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:43.028368622Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:43.028592093Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1955,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 310 310 310 F F F 1 020-039 176 176 176 309 309 309 T F T <--- 2 040-059 176 176 176 309 309 309 T F T <--- 3 060-079 178 178 178 310 310 310 T F T <--- 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 180 180 180 313 313 313 T F T <--- 8 160-179 179 179 179 312 312 312 T F T <--- 9 180-199 179 179 179 312 312 312 T F T <--- 10 200-219 176 176 176 309 309 309 F F F 11 220-239 180 180 180 313 313 313 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 178 178 178 310 310 310 F F F 14 280-299 174 174 174 306 306 306 T F T <--- 15 300-319 178 178 178 311 311 311 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 176 176 176 309 309 309 T F T <--- 18 360-379 176 176 176 309 309 309 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 178 178 178 310 310 310 F F F 21 420-439 180 179 180 315 312 315 F F F <--- 22 440-459 180 176 180 314 309 314 T F T <--- 23 460-479 180 175 180 314 308 314 T F T <--- 24 480-499 175 175 175 308 308 308 T F T <--- 25 500-519 178 178 178 310 310 310 F F F 26 520-539 180 178 180 314 310 314 F F F <--- 27 540-559 180 179 180 314 312 314 F F F <--- 28 560-579 180 176 180 314 309 314 T F T <--- 29 580-599 178 178 178 310 310 310 T F T <--- Max gen: 180, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 310 310 310 F F F 1 020-039 181 181 181 316 316 316 F F F 2 040-059 181 181 181 316 316 316 F F F 3 060-079 181 181 181 316 316 316 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 178 178 178 310 310 310 F F F 7 140-159 181 181 181 316 316 316 F F F 8 160-179 181 181 181 316 316 316 F F F 9 180-199 181 181 181 316 316 316 F F F 10 200-219 176 176 176 309 309 309 F F F 11 220-239 180 180 180 313 313 313 F F F 12 240-259 174 174 174 307 307 307 F F F 13 260-279 178 178 178 310 310 310 F F F 14 280-299 181 181 181 316 316 316 F F F 15 300-319 178 178 178 311 311 311 F F F 16 320-339 173 173 173 301 301 301 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 176 176 176 309 309 309 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 178 178 178 310 310 310 F F F 21 420-439 180 180 180 315 315 315 F F F 22 440-459 181 181 181 316 316 316 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 178 178 178 310 310 310 F F F 26 520-539 180 180 180 314 314 314 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 181 181 181 316 316 316 F F F 29 580-599 181 181 181 316 316 316 F F F Max gen: 181, Max flush: 316 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.326778623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.335671763Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.335982723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336014579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336024127Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336036726Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Crucible 53d5f2ae-7c00-4013-9a14-db47d6df7819 has session id: 4563cfb6-5b54-4be6-98b9-81958088a449","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336422906Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Upstairs opts: Upstairs UUID: 53d5f2ae-7c00-4013-9a14-db47d6df7819, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336442353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Crucible stats registered with UUID: 53d5f2ae-7c00-4013-9a14-db47d6df7819","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336453752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:57.33646436Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336642047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336659885Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336670964Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336821894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336847501Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"53d5f2ae-7c00-4013-9a14-db47d6df7819 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336858549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336868598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"0","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336879617Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"1","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336890235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"2","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336905283Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336916362Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336950538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"ds_connection connected from Ok(127.0.0.1:64919)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336976664Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"ds_connection connected from Ok(127.0.0.1:34312)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.336989163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"ds_connection connected from Ok(127.0.0.1:34667)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.337044455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.337664465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"0","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.337679633Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"1","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.337835753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"2","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338139814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[0]R flush_numbers[0..12]: [310, 316, 316, 316, 310, 301, 310, 316, 316, 316, 309, 313]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338178359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[0]R generation[0..12]: [178, 181, 181, 181, 178, 173, 178, 181, 181, 181, 176, 180]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338190687Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338202716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[1]R flush_numbers[0..12]: [310, 316, 316, 316, 310, 301, 310, 316, 316, 316, 309, 313]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338214504Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[1]R generation[0..12]: [178, 181, 181, 181, 178, 173, 178, 181, 181, 181, 176, 180]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338225973Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338237981Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[2]R flush_numbers[0..12]: [310, 316, 316, 316, 310, 301, 310, 316, 316, 316, 309, 313]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.33824978Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[2]R generation[0..12]: [178, 181, 181, 181, 178, 173, 178, 181, 181, 181, 176, 180]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338264128Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338276296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338286665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338297533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338307922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338318081Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"":"downstairs","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338328829Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338338598Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"0","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338349117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"1","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338359855Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"client":"2","":"downstairs","session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338370554Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338380613Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"53d5f2ae-7c00-4013-9a14-db47d6df7819 is now active with session: 4563cfb6-5b54-4be6-98b9-81958088a449","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338391621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.33840125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"session_id":"4563cfb6-5b54-4be6-98b9-81958088a449"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338411429Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Activated sub_volume 53d5f2ae-7c00-4013-9a14-db47d6df7819","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338420407Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338430426Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.338747445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"task":"crutest"} 01/30 Write block 135 len 32768 data: 14 14 15 14 15 12 9 12 02/30 Read block 507 len 8192 03/30 Flush 04/30 Write block 6 len 40960 data: 12 15 15 17 15 19 14 12 12 14 05/30 Read block 218 len 36864 06/30 Write block 253 len 40960 data: 15 11 7 9 8 7 8 10 8 10 07/30 Read block 350 len 40960 08/30 Read block 404 len 12288 09/30 Read block 165 len 4096 10/30 Write block 334 len 4096 data: 13 11/30 Write block 174 len 12288 data: 14 17 18 12/30 Write block 520 len 32768 data: 20 18 20 18 17 17 17 21 13/30 Read block 595 len 12288 14/30 Write block 153 len 4096 data: 14 15/30 Read block 371 len 24576 16/30 Read block 487 len 4096 17/30 Write block 537 len 12288 data: 20 21 19 18/30 Write block 361 len 16384 data: 24 29 27 28 19/30 Write block 305 len 4096 data: 13 20/30 Read block 542 len 16384 21/30 Write block 35 len 32768 data: 11 13 10 11 11 15 15 16 22/30 Write block 570 len 36864 data: 16 15 16 18 15 16 18 17 16 23/30 Read block 213 len 4096 24/30 Write block 42 len 12288 data: 17 16 18 25/30 Flush 26/30 Write block 225 len 4096 data: 18 27/30 Read block 269 len 8192 28/30 Read block 129 len 36864 29/30 Read block 408 len 36864 30/30 Write block 367 len 8192 data: 23 18 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 2 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:07:57.34814511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:07:57.348369511Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1964,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 178 318 318 310 F F F <--- 1 020-039 182 182 181 318 318 316 F F F <--- 2 040-059 182 182 181 318 318 316 F F F <--- 3 060-079 181 181 181 316 316 316 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 182 182 178 317 317 310 F F T <--- 7 140-159 182 182 181 318 318 316 F F T <--- 8 160-179 182 182 181 318 318 316 F F F <--- 9 180-199 181 181 181 316 316 316 F F F 10 200-219 176 176 176 309 309 309 F F F 11 220-239 180 180 180 313 313 313 T T F <--- 12 240-259 182 182 174 318 318 307 F F F <--- 13 260-279 182 182 178 318 318 310 F F F <--- 14 280-299 181 181 181 316 316 316 F F F 15 300-319 182 182 178 318 318 311 F F F <--- 16 320-339 182 182 173 318 318 301 F F F <--- 17 340-359 181 181 181 316 316 316 F F F 18 360-379 182 182 176 318 318 309 T T F <--- 19 380-399 168 168 168 295 295 295 F F F 20 400-419 178 178 178 310 310 310 F F F 21 420-439 180 180 180 315 315 315 F F F 22 440-459 181 181 181 316 316 316 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 178 178 178 310 310 310 F F F 26 520-539 182 182 180 318 318 314 F F F <--- 27 540-559 180 180 180 314 314 314 F F F 28 560-579 182 182 181 318 318 316 F F F <--- 29 580-599 181 181 181 316 316 316 F F F Max gen: 182, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 182 182 182 318 318 318 F F F 2 040-059 182 182 182 318 318 318 F F F 3 060-079 181 181 181 316 316 316 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 182 182 182 317 317 317 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 182 182 182 318 318 318 F F F 9 180-199 181 181 181 316 316 316 F F F 10 200-219 176 176 176 309 309 309 F F F 11 220-239 183 183 183 319 319 319 F F F 12 240-259 182 182 182 318 318 318 F F F 13 260-279 182 182 182 318 318 318 F F F 14 280-299 181 181 181 316 316 316 F F F 15 300-319 182 182 182 318 318 318 F F F 16 320-339 182 182 182 318 318 318 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 183 183 183 319 319 319 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 178 178 178 310 310 310 F F F 21 420-439 180 180 180 315 315 315 F F F 22 440-459 181 181 181 316 316 316 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 178 178 178 310 310 310 F F F 26 520-539 182 182 182 318 318 318 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 182 182 182 318 318 318 F F F 29 580-599 181 181 181 316 316 316 F F F Max gen: 183, Max flush: 319 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.631242441Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.639963123Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640276663Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640312858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640322847Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640335555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Crucible 66cfb111-d9bc-4c83-9ba5-5741efab9ee8 has session id: f0970602-852f-4873-abb8-d9bc84f478aa","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640731374Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Upstairs opts: Upstairs UUID: 66cfb111-d9bc-4c83-9ba5-5741efab9ee8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640749831Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Crucible stats registered with UUID: 66cfb111-d9bc-4c83-9ba5-5741efab9ee8","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.64076061Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:11.640770779Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640906861Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640924669Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.640937407Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641117904Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641136111Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"66cfb111-d9bc-4c83-9ba5-5741efab9ee8 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641167947Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641178096Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"0","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641188665Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"1","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641199053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"2","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641209582Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641220291Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641231079Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:43981)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641283242Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:39408)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641310909Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:42979)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.641323707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642023637Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"1","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642044684Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"0","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642061392Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"2","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642388729Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 318, 316, 310, 301, 317, 318, 318, 316, 309, 319]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642410447Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[0]R generation[0..12]: [182, 182, 182, 181, 178, 173, 182, 182, 182, 181, 176, 183]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642422075Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642433844Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 318, 316, 310, 301, 317, 318, 318, 316, 309, 319]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642445302Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[1]R generation[0..12]: [182, 182, 182, 181, 178, 173, 182, 182, 182, 181, 176, 183]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642456621Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642468379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 318, 316, 310, 301, 317, 318, 318, 316, 309, 319]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642479688Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[2]R generation[0..12]: [182, 182, 182, 181, 178, 173, 182, 182, 182, 181, 176, 183]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642490856Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642502365Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642512583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642522922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642532881Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642544129Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"":"downstairs","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642554808Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642564167Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"0","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642574675Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"1","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642585164Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"client":"2","":"downstairs","session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642595933Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642605791Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"66cfb111-d9bc-4c83-9ba5-5741efab9ee8 is now active with session: f0970602-852f-4873-abb8-d9bc84f478aa","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.64261629Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642625799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"session_id":"f0970602-852f-4873-abb8-d9bc84f478aa"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642635577Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Activated sub_volume 66cfb111-d9bc-4c83-9ba5-5741efab9ee8","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642644216Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642657325Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.642935959Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"task":"crutest"} 01/30 Write block 584 len 12288 data: 13 12 15 02/30 Read block 241 len 36864 03/30 Write block 253 len 16384 data: 16 12 8 10 04/30 Write block 518 len 4096 data: 15 05/30 Write block 65 len 4096 data: 12 06/30 Read block 361 len 12288 07/30 Read block 93 len 20480 08/30 Read block 250 len 40960 09/30 Read block 277 len 36864 10/30 Read block 441 len 24576 11/30 Read block 559 len 8192 12/30 Write block 55 len 28672 data: 14 8 8 8 8 9 13 13/30 Read block 90 len 8192 14/30 Read block 14 len 36864 15/30 Write block 380 len 24576 data: 15 17 18 15 14 15 16/30 Read block 197 len 4096 17/30 Write block 202 len 4096 data: 18 18/30 Write block 186 len 16384 data: 19 15 15 14 19/30 Write block 416 len 36864 data: 12 16 16 14 12 12 12 11 9 20/30 Read block 268 len 36864 21/30 Write block 385 len 12288 data: 16 14 13 22/30 Flush 23/30 Write block 372 len 32768 data: 10 9 10 7 8 10 11 14 24/30 Read block 334 len 28672 25/30 Write block 169 len 24576 data: 15 14 17 14 15 15 26/30 Write block 64 len 12288 data: 12 13 13 27/30 Write block 27 len 8192 data: 12 11 28/30 Write block 40 len 36864 data: 16 16 18 17 19 16 16 22 21 29/30 Read block 327 len 8192 30/30 Write block 209 len 32768 data: 10 8 5 5 9 8 12 15 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 4 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:11.651649232Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:11.651858455Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1973,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 182 182 182 318 318 318 T T F <--- 2 040-059 184 184 182 320 320 318 T T F <--- 3 060-079 184 184 181 320 320 316 T T F <--- 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 182 182 182 317 317 317 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 182 182 182 318 318 318 T T F <--- 9 180-199 184 184 181 320 320 316 F F F <--- 10 200-219 184 184 176 320 320 309 T T F <--- 11 220-239 183 183 183 319 319 319 F F F 12 240-259 184 184 182 320 320 318 F F F <--- 13 260-279 182 182 182 318 318 318 F F F 14 280-299 181 181 181 316 316 316 F F F 15 300-319 182 182 182 318 318 318 F F F 16 320-339 182 182 182 318 318 318 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 183 183 183 319 319 319 T T F <--- 19 380-399 184 184 168 320 320 295 F F F <--- 20 400-419 184 184 178 320 320 310 F F F <--- 21 420-439 184 184 180 320 320 315 F F F <--- 22 440-459 181 181 181 316 316 316 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 184 178 320 320 310 F F F <--- 26 520-539 182 182 182 318 318 318 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 182 182 182 318 318 318 F F F 29 580-599 184 184 181 320 320 316 F F T <--- Max gen: 184, Max flush: 320 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 185 185 185 321 321 321 F F F 2 040-059 185 185 185 321 321 321 F F F 3 060-079 185 185 185 321 321 321 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 182 182 182 317 317 317 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 185 185 185 321 321 321 F F F 9 180-199 184 184 184 320 320 320 F F F 10 200-219 185 185 185 321 321 321 F F F 11 220-239 183 183 183 319 319 319 F F F 12 240-259 184 184 184 320 320 320 F F F 13 260-279 182 182 182 318 318 318 F F F 14 280-299 181 181 181 316 316 316 F F F 15 300-319 182 182 182 318 318 318 F F F 16 320-339 182 182 182 318 318 318 F F F 17 340-359 181 181 181 316 316 316 F F F 18 360-379 185 185 185 321 321 321 F F F 19 380-399 184 184 184 320 320 320 F F F 20 400-419 184 184 184 320 320 320 F F F 21 420-439 184 184 184 320 320 320 F F F 22 440-459 181 181 181 316 316 316 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 184 184 320 320 320 F F F 26 520-539 182 182 182 318 318 318 F F F 27 540-559 180 180 180 314 314 314 F F F 28 560-579 182 182 182 318 318 318 F F F 29 580-599 184 184 184 320 320 320 F F F Max gen: 185, Max flush: 321 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.926826007Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.935592053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.935912522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.935943218Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.935954646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.935967345Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Crucible 85135c39-4bbf-4a17-98b0-bfeeeb02c6d5 has session id: 290f862e-caa4-4465-88af-31b2bf76135b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936385161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Upstairs opts: Upstairs UUID: 85135c39-4bbf-4a17-98b0-bfeeeb02c6d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936416287Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Crucible stats registered with UUID: 85135c39-4bbf-4a17-98b0-bfeeeb02c6d5","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936428475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:25.936438574Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936597533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936631579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936643467Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936782979Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936793228Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"85135c39-4bbf-4a17-98b0-bfeeeb02c6d5 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936831583Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936841792Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"0","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.93685222Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"1","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936865468Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"2","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936875987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936886926Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936897704Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63589)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936967425Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36486)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.936986913Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57753)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.937012449Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.937733646Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"0","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.937761043Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"1","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.937903454Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"2","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938194607Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[0]R flush_numbers[0..12]: [318, 321, 321, 321, 310, 301, 317, 318, 321, 320, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938219343Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[0]R generation[0..12]: [182, 185, 185, 185, 178, 173, 182, 182, 185, 184, 185, 183]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938231942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.93824386Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[1]R flush_numbers[0..12]: [318, 321, 321, 321, 310, 301, 317, 318, 321, 320, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938255579Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[1]R generation[0..12]: [182, 185, 185, 185, 178, 173, 182, 182, 185, 184, 185, 183]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938269177Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938281125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[2]R flush_numbers[0..12]: [318, 321, 321, 321, 310, 301, 317, 318, 321, 320, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938292534Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[2]R generation[0..12]: [182, 185, 185, 185, 178, 173, 182, 182, 185, 184, 185, 183]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938304012Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938315741Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.93832595Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938336548Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938346687Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938356846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"":"downstairs","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938367394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938376963Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"0","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938387512Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"1","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.93839799Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"client":"2","":"downstairs","session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938408549Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938418518Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"85135c39-4bbf-4a17-98b0-bfeeeb02c6d5 is now active with session: 290f862e-caa4-4465-88af-31b2bf76135b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938430536Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938440135Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"session_id":"290f862e-caa4-4465-88af-31b2bf76135b"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938449984Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Activated sub_volume 85135c39-4bbf-4a17-98b0-bfeeeb02c6d5","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938458712Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938467601Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.938806017Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"task":"crutest"} 01/30 Write block 292 len 24576 data: 11 10 10 11 10 9 02/30 Write block 581 len 12288 data: 14 13 14 03/30 Read block 248 len 28672 04/30 Write block 124 len 8192 data: 16 17 05/30 Write block 452 len 4096 data: 12 06/30 Read block 50 len 40960 07/30 Write block 350 len 32768 data: 17 15 14 16 17 19 16 15 08/30 Read block 168 len 32768 09/30 Read block 277 len 28672 10/30 Write block 348 len 24576 data: 12 14 18 16 15 17 11/30 Write block 72 len 24576 data: 8 8 9 10 9 8 12/30 Write block 255 len 36864 data: 9 11 9 8 9 11 9 11 10 13/30 Write block 526 len 32768 data: 18 22 20 17 16 16 13 14 14/30 Write block 386 len 12288 data: 15 14 14 15/30 Write block 56 len 24576 data: 9 9 9 9 10 14 16/30 Write block 544 len 12288 data: 14 14 14 17/30 Read block 511 len 40960 18/30 Read block 532 len 40960 19/30 Flush 20/30 Write block 197 len 20480 data: 9 11 11 14 13 21/30 Write block 554 len 20480 data: 11 11 12 11 13 22/30 Write block 131 len 12288 data: 14 15 17 23/30 Read block 182 len 36864 24/30 Write block 227 len 16384 data: 18 15 10 8 25/30 Write block 420 len 16384 data: 13 13 13 12 26/30 Read block 130 len 20480 27/30 Read block 571 len 20480 28/30 Write block 310 len 4096 data: 17 29/30 Read block 45 len 8192 30/30 Write block 292 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 6 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 16 18 Done 29 29 14 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:25.947044172Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:25.947240157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1982,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 185 185 185 321 321 321 F F F 2 040-059 186 186 185 322 322 321 F F F <--- 3 060-079 186 186 185 322 322 321 F F T <--- 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 186 186 182 322 322 317 T T T <--- 7 140-159 182 182 182 318 318 318 F F F 8 160-179 185 185 185 321 321 321 F F F 9 180-199 184 184 184 320 320 320 T T F <--- 10 200-219 185 185 185 321 321 321 T T F <--- 11 220-239 183 183 183 319 319 319 T T F <--- 12 240-259 186 186 184 322 322 320 F F T <--- 13 260-279 186 186 182 322 322 318 F F T <--- 14 280-299 186 186 181 322 322 316 T T T <--- 15 300-319 182 182 182 318 318 318 T T F <--- 16 320-339 182 182 182 318 318 318 F F F 17 340-359 186 186 181 322 322 316 F F T <--- 18 360-379 185 185 185 321 321 321 F F F 19 380-399 186 186 184 322 322 320 F F T <--- 20 400-419 184 184 184 320 320 320 F F F 21 420-439 184 184 184 320 320 320 T T F <--- 22 440-459 186 186 181 322 322 316 F F T <--- 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 184 184 320 320 320 F F F 26 520-539 186 186 182 322 322 318 F F T <--- 27 540-559 186 186 180 322 322 314 T T F <--- 28 560-579 182 182 182 318 318 318 F F F 29 580-599 186 186 184 322 322 320 F F T <--- Max gen: 186, Max flush: 322 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 185 185 185 321 321 321 F F F 2 040-059 186 186 186 322 322 322 F F F 3 060-079 186 186 186 322 322 322 F F F 4 080-099 178 178 178 310 310 310 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 187 187 187 323 323 323 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 185 185 185 321 321 321 F F F 9 180-199 187 187 187 323 323 323 F F F 10 200-219 187 187 187 323 323 323 F F F 11 220-239 187 187 187 323 323 323 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 186 186 186 322 322 322 F F F 14 280-299 187 187 187 323 323 323 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 182 182 182 318 318 318 F F F 17 340-359 186 186 186 322 322 322 F F F 18 360-379 185 185 185 321 321 321 F F F 19 380-399 186 186 186 322 322 322 F F F 20 400-419 184 184 184 320 320 320 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 184 184 320 320 320 F F F 26 520-539 186 186 186 322 322 322 F F F 27 540-559 187 187 187 323 323 323 F F F 28 560-579 182 182 182 318 318 318 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 187, Max flush: 323 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.229254238Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.237986299Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238333724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.23836171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238377498Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238389977Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Crucible 87bfe08c-3da9-4284-bea4-0825b7180fe4 has session id: 91dd76a4-562e-4cf3-b6cf-3806185fbe2c","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238802783Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Upstairs opts: Upstairs UUID: 87bfe08c-3da9-4284-bea4-0825b7180fe4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238820991Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Crucible stats registered with UUID: 87bfe08c-3da9-4284-bea4-0825b7180fe4","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238836559Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:40.238846338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.238995148Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239027224Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239039013Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239195742Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239205951Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"87bfe08c-3da9-4284-bea4-0825b7180fe4 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239225089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239234707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"0","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239245076Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"1","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239255355Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"2","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239265643Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239276352Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239306858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39082)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239318037Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36097)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239354552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60880)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.239424723Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240012637Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"1","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240039533Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"0","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240052112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"2","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240342774Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[0]R flush_numbers[0..12]: [318, 321, 322, 322, 310, 301, 323, 318, 321, 323, 323, 323]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.24037246Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[0]R generation[0..12]: [182, 185, 186, 186, 178, 173, 187, 182, 185, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240384809Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240396767Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[1]R flush_numbers[0..12]: [318, 321, 322, 322, 310, 301, 323, 318, 321, 323, 323, 323]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240408086Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[1]R generation[0..12]: [182, 185, 186, 186, 178, 173, 187, 182, 185, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240419274Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240430903Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[2]R flush_numbers[0..12]: [318, 321, 322, 322, 310, 301, 323, 318, 321, 323, 323, 323]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240442271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[2]R generation[0..12]: [182, 185, 186, 186, 178, 173, 187, 182, 185, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.24045372Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240467438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240477557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240488185Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240498404Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240508573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"":"downstairs","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240518931Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.24052826Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"0","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240538359Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"1","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240548717Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"client":"2","":"downstairs","session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240559526Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240569315Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"87bfe08c-3da9-4284-bea4-0825b7180fe4 is now active with session: 91dd76a4-562e-4cf3-b6cf-3806185fbe2c","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240580013Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240589412Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"session_id":"91dd76a4-562e-4cf3-b6cf-3806185fbe2c"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240599301Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Activated sub_volume 87bfe08c-3da9-4284-bea4-0825b7180fe4","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240608Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240617248Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.240968243Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"task":"crutest"} 01/30 Write block 590 len 4096 data: 15 02/30 Write block 86 len 40960 data: 11 11 12 9 10 10 10 11 12 12 03/30 Read block 439 len 8192 04/30 Write block 555 len 32768 data: 12 13 12 14 15 17 19 19 05/30 Read block 44 len 32768 06/30 Flush 07/30 Flush 08/30 Write block 235 len 24576 data: 13 11 11 12 11 11 09/30 Read block 311 len 24576 10/30 Write block 533 len 36864 data: 15 13 18 20 21 22 20 22 21 11/30 Write block 163 len 40960 data: 15 14 13 13 14 16 16 15 18 15 12/30 Read block 510 len 32768 13/30 Write block 401 len 16384 data: 9 12 13 11 14/30 Write block 161 len 12288 data: 19 18 16 15/30 Read block 187 len 8192 16/30 Read block 87 len 28672 17/30 Read block 497 len 28672 18/30 Write block 267 len 16384 data: 12 14 14 17 19/30 Write block 123 len 28672 data: 15 17 18 15 12 16 17 20/30 Flush 21/30 Write block 341 len 40960 data: 10 10 10 11 14 14 13 13 15 19 22/30 Write block 83 len 24576 data: 8 11 12 12 12 13 23/30 Write block 331 len 16384 data: 14 15 15 14 24/30 Write block 251 len 24576 data: 13 12 17 13 10 12 25/30 Flush 26/30 Write block 120 len 16384 data: 12 15 15 16 27/30 Write block 68 len 20480 data: 13 11 9 9 9 28/30 Read block 64 len 20480 29/30 Read block 97 len 24576 30/30 Read block 219 len 8192 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 2 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:40.250212007Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:40.250412102Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":1991,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 185 185 185 321 321 321 F F F 2 040-059 186 186 186 322 322 322 F F F 3 060-079 186 186 186 322 322 322 T T F <--- 4 080-099 188 188 178 327 327 310 F F F <--- 5 100-119 173 173 173 301 301 301 F F F 6 120-139 188 188 187 326 326 323 T T F <--- 7 140-159 182 182 182 318 318 318 F F F 8 160-179 188 188 185 326 326 321 F F F <--- 9 180-199 187 187 187 323 323 323 F F F 10 200-219 187 187 187 323 323 323 F F F 11 220-239 188 188 187 326 326 323 F F F <--- 12 240-259 188 188 186 327 327 322 F F F <--- 13 260-279 188 188 186 326 326 322 F F F <--- 14 280-299 187 187 187 323 323 323 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 188 188 182 327 327 318 F F F <--- 17 340-359 188 188 186 327 327 322 F F F <--- 18 360-379 185 185 185 321 321 321 F F F 19 380-399 186 186 186 322 322 322 F F F 20 400-419 188 188 184 326 326 320 F F F <--- 21 420-439 187 187 187 323 323 323 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 184 184 320 320 320 F F F 26 520-539 188 188 186 326 326 322 F F F <--- 27 540-559 188 188 187 326 326 323 F F F <--- 28 560-579 188 188 182 324 324 318 F F F <--- 29 580-599 188 188 186 324 324 322 F F F <--- Max gen: 188, Max flush: 327 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 185 185 185 321 321 321 F F F 2 040-059 186 186 186 322 322 322 F F F 3 060-079 189 189 189 328 328 328 F F F 4 080-099 188 188 188 327 327 327 F F F 5 100-119 173 173 173 301 301 301 F F F 6 120-139 189 189 189 328 328 328 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 188 188 188 326 326 326 F F F 9 180-199 187 187 187 323 323 323 F F F 10 200-219 187 187 187 323 323 323 F F F 11 220-239 188 188 188 326 326 326 F F F 12 240-259 188 188 188 327 327 327 F F F 13 260-279 188 188 188 326 326 326 F F F 14 280-299 187 187 187 323 323 323 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 188 188 188 327 327 327 F F F 17 340-359 188 188 188 327 327 327 F F F 18 360-379 185 185 185 321 321 321 F F F 19 380-399 186 186 186 322 322 322 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 184 184 320 320 320 F F F 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 188 188 324 324 324 F F F 29 580-599 188 188 188 324 324 324 F F F Max gen: 189, Max flush: 328 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.523031888Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:54.530234307Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.530290439Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.537373413Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.537661716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.537698071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.53770788Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.537720338Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Crucible 5e4418a9-6562-412e-9bb4-d86e7916ead8 has session id: 402441e3-0252-4133-be8e-45046be70e7b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538029708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Upstairs opts: Upstairs UUID: 5e4418a9-6562-412e-9bb4-d86e7916ead8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538071053Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Crucible stats registered with UUID: 5e4418a9-6562-412e-9bb4-d86e7916ead8","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538084901Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:54.53809434Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538230772Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"2","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.53824942Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"0","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538259739Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"1","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538417968Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538432326Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"5e4418a9-6562-412e-9bb4-d86e7916ead8 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538467982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.53847758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"0","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538487379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"1","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538497198Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"2","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538506887Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"2","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538517065Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"0","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538527014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"1","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51024)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538576758Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"2","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39040)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538601834Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"1","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51256)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.538613333Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"io task","client":"0","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539197837Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"2","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539241322Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"1","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539260529Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"0","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.53956235Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[0]R flush_numbers[0..12]: [318, 321, 322, 328, 327, 301, 328, 318, 326, 323, 323, 326]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539596716Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[0]R generation[0..12]: [182, 185, 186, 189, 188, 173, 189, 182, 188, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539608054Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539619183Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[1]R flush_numbers[0..12]: [318, 321, 322, 328, 327, 301, 328, 318, 326, 323, 323, 326]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539629741Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[1]R generation[0..12]: [182, 185, 186, 189, 188, 173, 189, 182, 188, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.53964022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539651189Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[2]R flush_numbers[0..12]: [318, 321, 322, 328, 327, 301, 328, 318, 326, 323, 323, 326]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539661737Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[2]R generation[0..12]: [182, 185, 186, 189, 188, 173, 189, 182, 188, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539672396Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539683405Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539692843Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539702772Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539712181Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539723099Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"":"downstairs","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539732828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539741587Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"0","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539751216Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"1","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539760785Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"client":"2","":"downstairs","session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539770623Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539779652Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"5e4418a9-6562-412e-9bb4-d86e7916ead8 is now active with session: 402441e3-0252-4133-be8e-45046be70e7b","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539789521Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.53979828Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"session_id":"402441e3-0252-4133-be8e-45046be70e7b"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539807438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Activated sub_volume 5e4418a9-6562-412e-9bb4-d86e7916ead8","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539815377Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.539823406Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.540141995Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} 01/30 Write block 219 len 8192 data: 22 21 02/30 Write block 27 len 8192 data: 13 12 03/30 Read block 453 len 32768 04/30 Read block 106 len 12288 05/30 Write block 40 len 32768 data: 17 17 19 18 20 17 17 23 06/30 Write block 332 len 40960 data: 16 16 15 12 9 8 7 8 10 11 07/30 Write block 590 len 16384 data: 16 13 16 17 08/30 Write block 515 len 12288 data: 12 12 17 09/30 Read block 574 len 20480 10/30 Read block 172 len 36864 11/30 Read block 506 len 36864 12/30 Read block 495 len 20480 13/30 Write block 22 len 32768 data: 11 13 15 13 14 14 13 11 14/30 Write block 178 len 40960 data: 20 14 15 13 14 19 18 17 20 16 15/30 Read block 312 len 24576 16/30 Read block 160 len 4096 17/30 Flush 18/30 Write block 109 len 20480 data: 15 13 11 9 13 19/30 Write block 377 len 16384 data: 11 12 15 16 20/30 Write block 195 len 24576 data: 9 10 10 12 12 15 21/30 Read block 322 len 24576 22/30 Read block 228 len 32768 23/30 Read block 449 len 36864 24/30 Read block 141 len 20480 25/30 Write block 208 len 28672 data: 9 11 9 6 6 10 9 26/30 Read block 566 len 4096 27/30 Write block 399 len 36864 data: 11 9 10 13 14 12 16 15 16 28/30 Read block 504 len 32768 29/30 Read block 128 len 24576 30/30 Read block 588 len 32768 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:08:54.549249437Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:08:54.549451271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2000,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 185 190 190 321 329 329 F F F <--- 2 040-059 186 190 190 322 329 329 F F F <--- 3 060-079 189 189 189 328 328 328 F F F 4 080-099 188 188 188 327 327 327 F F F 5 100-119 173 173 173 301 301 301 F T T <--- 6 120-139 189 189 189 328 328 328 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 188 190 190 326 329 329 F F F <--- 9 180-199 187 190 190 323 329 329 F T T <--- 10 200-219 187 190 190 323 329 329 F T T <--- 11 220-239 188 190 190 326 329 329 F F F <--- 12 240-259 188 188 188 327 327 327 F F F 13 260-279 188 188 188 326 326 326 F F F 14 280-299 187 187 187 323 323 323 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 188 190 190 327 329 329 F F F <--- 17 340-359 188 190 190 327 329 329 F F F <--- 18 360-379 185 185 185 321 321 321 F T T <--- 19 380-399 186 186 186 322 322 322 F T T <--- 20 400-419 188 188 188 326 326 326 F T T <--- 21 420-439 187 187 187 323 323 323 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 184 190 190 320 329 329 F F F <--- 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 188 188 324 324 324 F F F 29 580-599 188 190 190 324 329 329 F F F <--- Max gen: 190, Max flush: 329 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 189 189 189 328 328 328 F F F 4 080-099 188 188 188 327 327 327 F F F 5 100-119 191 191 191 330 330 330 F F F 6 120-139 189 189 189 328 328 328 F F F 7 140-159 182 182 182 318 318 318 F F F 8 160-179 190 190 190 329 329 329 F F F 9 180-199 191 191 191 330 330 330 F F F 10 200-219 191 191 191 330 330 330 F F F 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 327 327 327 F F F 13 260-279 188 188 188 326 326 326 F F F 14 280-299 187 187 187 323 323 323 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 190 190 190 329 329 329 F F F 17 340-359 190 190 190 329 329 329 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 191 191 191 330 330 330 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 181 181 316 316 316 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 188 188 324 324 324 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 191, Max flush: 330 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.81757165Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:08.824630297Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.82468321Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.831381224Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.831669336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.831699752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.831708041Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83171967Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Crucible 128d4660-d5e2-4b7e-acc6-8e7f9e9ef8ed has session id: 62934761-d4ac-4f15-9ed8-6185218cee18","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832083123Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Upstairs opts: Upstairs UUID: 128d4660-d5e2-4b7e-acc6-8e7f9e9ef8ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832129517Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Crucible stats registered with UUID: 128d4660-d5e2-4b7e-acc6-8e7f9e9ef8ed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832144125Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:08.832153584Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832291826Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"2","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832323522Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"1","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83233453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"0","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832472312Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832485531Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"128d4660-d5e2-4b7e-acc6-8e7f9e9ef8ed active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832520366Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832531895Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"0","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832541683Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"1","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832551282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"2","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832560821Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"2","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83257067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"0","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832580678Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"1","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"ds_connection connected from Ok(127.0.0.1:33378)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832660948Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"2","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"ds_connection connected from Ok(127.0.0.1:50461)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832747987Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"0","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"ds_connection connected from Ok(127.0.0.1:60700)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.832766714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"io task","client":"1","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833461884Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"0","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833474173Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"1","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833486171Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"2","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83380573Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[0]R flush_numbers[0..12]: [318, 329, 329, 328, 327, 330, 328, 318, 329, 330, 330, 329]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833826057Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[0]R generation[0..12]: [182, 190, 190, 189, 188, 191, 189, 182, 190, 191, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833838466Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833849604Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[1]R flush_numbers[0..12]: [318, 329, 329, 328, 327, 330, 328, 318, 329, 330, 330, 329]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833860193Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[1]R generation[0..12]: [182, 190, 190, 189, 188, 191, 189, 182, 190, 191, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833870752Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83388175Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[2]R flush_numbers[0..12]: [318, 329, 329, 328, 327, 330, 328, 318, 329, 330, 330, 329]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833892379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[2]R generation[0..12]: [182, 190, 190, 189, 188, 191, 189, 182, 190, 191, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833903027Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833914026Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833923615Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833933353Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833942702Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833952001Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"":"downstairs","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83396163Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833970379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"0","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833980067Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"1","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.833990896Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"client":"2","":"downstairs","session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834000655Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834009894Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"128d4660-d5e2-4b7e-acc6-8e7f9e9ef8ed is now active with session: 62934761-d4ac-4f15-9ed8-6185218cee18","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834019872Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834028701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"session_id":"62934761-d4ac-4f15-9ed8-6185218cee18"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.83403794Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Activated sub_volume 128d4660-d5e2-4b7e-acc6-8e7f9e9ef8ed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834046089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834054138Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.834384275Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} 01/30 Write block 117 len 24576 data: 12 11 11 13 16 16 02/30 Write block 446 len 36864 data: 14 11 12 13 12 10 13 11 12 03/30 Write block 123 len 28672 data: 17 18 19 16 13 17 18 04/30 Read block 169 len 12288 05/30 Write block 213 len 16384 data: 11 10 13 16 06/30 Write block 446 len 20480 data: 15 12 13 14 13 07/30 Write block 363 len 16384 data: 28 29 25 24 08/30 Read block 425 len 40960 09/30 Write block 136 len 4096 data: 15 10/30 Flush 11/30 Write block 117 len 24576 data: 13 12 12 14 17 17 12/30 Read block 534 len 20480 13/30 Read block 403 len 8192 14/30 Read block 230 len 8192 15/30 Read block 346 len 12288 16/30 Read block 63 len 4096 17/30 Write block 497 len 32768 data: 13 12 11 11 13 11 15 16 18/30 Flush 19/30 Write block 279 len 16384 data: 12 14 13 11 20/30 Write block 133 len 32768 data: 18 17 15 16 16 15 16 13 21/30 Write block 561 len 28672 data: 20 20 15 18 19 18 14 22/30 Flush 23/30 Write block 332 len 32768 data: 17 17 16 13 10 9 8 9 24/30 Read block 356 len 28672 25/30 Flush 26/30 Write block 76 len 20480 data: 10 9 11 12 11 27/30 Write block 187 len 36864 data: 17 16 15 13 15 16 12 7 10 28/30 Read block 507 len 32768 29/30 Write block 84 len 24576 data: 12 13 13 13 14 10 30/30 Read block 344 len 4096 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 4 30 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:08.842841071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:08.843043905Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2009,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 189 189 189 328 328 328 F T T <--- 4 080-099 188 188 188 327 327 327 F T T <--- 5 100-119 191 192 192 330 332 332 T F F <--- 6 120-139 189 192 192 328 333 333 T F F <--- 7 140-159 182 192 192 318 333 333 F F F <--- 8 160-179 190 190 190 329 329 329 F F F 9 180-199 191 191 191 330 330 330 F T T <--- 10 200-219 191 192 192 330 331 331 F F F <--- 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 327 327 327 F F F 13 260-279 188 192 192 326 333 333 F F F <--- 14 280-299 187 192 192 323 333 333 F F F <--- 15 300-319 187 187 187 323 323 323 F F F 16 320-339 190 192 192 329 334 334 F F F <--- 17 340-359 190 190 190 329 329 329 F F F 18 360-379 191 192 192 330 331 331 F F F <--- 19 380-399 191 191 191 330 330 330 F F F 20 400-419 191 191 191 330 330 330 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 186 192 192 322 331 331 T F F <--- 23 460-479 181 181 181 316 316 316 F F F 24 480-499 181 192 192 316 332 332 F F F <--- 25 500-519 190 192 192 329 332 332 F F F <--- 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 192 192 324 333 333 F F F <--- 29 580-599 190 190 190 329 329 329 F F F Max gen: 192, Max flush: 334 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 193 193 193 335 335 335 F F F 4 080-099 193 193 193 335 335 335 F F F 5 100-119 192 192 192 332 332 332 F F F 6 120-139 192 192 192 333 333 333 F F F 7 140-159 192 192 192 333 333 333 F F F 8 160-179 190 190 190 329 329 329 F F F 9 180-199 193 193 193 335 335 335 F F F 10 200-219 192 192 192 331 331 331 F F F 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 327 327 327 F F F 13 260-279 192 192 192 333 333 333 F F F 14 280-299 192 192 192 333 333 333 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 192 192 192 334 334 334 F F F 17 340-359 190 190 190 329 329 329 F F F 18 360-379 192 192 192 331 331 331 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 191 191 191 330 330 330 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 192 192 192 332 332 332 F F F 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 190 190 190 329 329 329 F F F Max gen: 193, Max flush: 335 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.108848034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117100696Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117397628Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117428754Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117437303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117449021Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Crucible 176184b1-e215-41ec-9b97-31967246b978 has session id: 8fe3f819-f2ff-4b98-ae35-9b65c511bccb","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117795746Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Upstairs opts: Upstairs UUID: 176184b1-e215-41ec-9b97-31967246b978, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117850009Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Crucible stats registered with UUID: 176184b1-e215-41ec-9b97-31967246b978","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.117876256Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:23.117890014Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118045704Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"0","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118062982Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"2","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118073281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"1","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118237329Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118252477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"176184b1-e215-41ec-9b97-31967246b978 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118292022Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118301401Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"0","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.1183112Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"1","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118321438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"2","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118331157Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"2","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118341376Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"0","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118352964Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"1","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48096)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118423285Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"2","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51687)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118492376Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"1","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"ds_connection connected from Ok(127.0.0.1:63399)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.118503755Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"io task","client":"0","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119107857Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"0","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119133453Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"1","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119165229Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"2","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119482728Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[0]R flush_numbers[0..12]: [318, 329, 329, 335, 335, 332, 333, 333, 329, 335, 331, 329]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119512194Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[0]R generation[0..12]: [182, 190, 190, 193, 193, 192, 192, 192, 190, 193, 192, 190]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119524223Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119535711Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[1]R flush_numbers[0..12]: [318, 329, 329, 335, 335, 332, 333, 333, 329, 335, 331, 329]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.11954677Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[1]R generation[0..12]: [182, 190, 190, 193, 193, 192, 192, 192, 190, 193, 192, 190]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119557929Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119569347Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[2]R flush_numbers[0..12]: [318, 329, 329, 335, 335, 332, 333, 333, 329, 335, 331, 329]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119582695Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[2]R generation[0..12]: [182, 190, 190, 193, 193, 192, 192, 192, 190, 193, 192, 190]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119593714Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119604922Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119614701Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.11962477Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119634469Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119644237Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"":"downstairs","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119654356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119663445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"0","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119673384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"1","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119683242Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"client":"2","":"downstairs","session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119693471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.11970303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"176184b1-e215-41ec-9b97-31967246b978 is now active with session: 8fe3f819-f2ff-4b98-ae35-9b65c511bccb","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119713438Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119722497Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"session_id":"8fe3f819-f2ff-4b98-ae35-9b65c511bccb"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119731946Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Activated sub_volume 176184b1-e215-41ec-9b97-31967246b978","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119741465Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.119749984Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.120083081Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"task":"crutest"} 01/30 Write block 583 len 28672 data: 15 14 13 16 14 14 14 02/30 Write block 542 len 24576 data: 21 17 15 15 15 13 03/30 Read block 335 len 40960 04/30 Write block 537 len 24576 data: 22 23 21 23 22 22 05/30 Read block 143 len 24576 06/30 Write block 573 len 12288 data: 19 16 17 07/30 Read block 394 len 16384 08/30 Read block 255 len 28672 09/30 Read block 201 len 28672 10/30 Read block 243 len 8192 11/30 Write block 597 len 4096 data: 7 12/30 Flush 13/30 Write block 54 len 12288 data: 15 15 10 14/30 Write block 141 len 36864 data: 10 13 11 15 14 13 11 13 14 15/30 Read block 31 len 8192 16/30 Read block 518 len 36864 17/30 Read block 459 len 24576 18/30 Read block 410 len 28672 19/30 Read block 130 len 20480 20/30 Read block 131 len 36864 21/30 Write block 321 len 8192 data: 8 7 22/30 Read block 389 len 24576 23/30 Read block 200 len 16384 24/30 Read block 527 len 20480 25/30 Flush 26/30 Write block 237 len 12288 data: 12 13 12 27/30 Read block 185 len 20480 28/30 Read block 488 len 20480 29/30 Write block 254 len 32768 data: 14 11 13 10 9 10 12 10 30/30 Read block 468 len 36864 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 10 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 28 29 Done 29 30 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:23.129361471Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:23.129576423Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2018,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 194 194 190 337 337 329 F F F <--- 3 060-079 193 193 193 335 335 335 F F F 4 080-099 193 193 193 335 335 335 F F F 5 100-119 192 192 192 332 332 332 F F F 6 120-139 192 192 192 333 333 333 F F F 7 140-159 194 194 192 337 337 333 F F F <--- 8 160-179 190 190 190 329 329 329 F F F 9 180-199 193 193 193 335 335 335 F F F 10 200-219 192 192 192 331 331 331 F F F 11 220-239 190 190 190 329 329 329 T T F <--- 12 240-259 188 188 188 327 327 327 T T F <--- 13 260-279 192 192 192 333 333 333 T T F <--- 14 280-299 192 192 192 333 333 333 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 194 194 192 337 337 334 F F F <--- 17 340-359 190 190 190 329 329 329 F F F 18 360-379 192 192 192 331 331 331 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 191 191 191 330 330 330 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 192 192 192 332 332 332 F F F 26 520-539 194 194 188 336 336 326 F F F <--- 27 540-559 194 194 188 336 336 326 F F T <--- 28 560-579 194 194 192 336 336 333 F F F <--- 29 580-599 194 194 190 336 336 329 F F T <--- Max gen: 194, Max flush: 337 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 194 194 194 337 337 337 F F F 3 060-079 193 193 193 335 335 335 F F F 4 080-099 193 193 193 335 335 335 F F F 5 100-119 192 192 192 332 332 332 F F F 6 120-139 192 192 192 333 333 333 F F F 7 140-159 194 194 194 337 337 337 F F F 8 160-179 190 190 190 329 329 329 F F F 9 180-199 193 193 193 335 335 335 F F F 10 200-219 192 192 192 331 331 331 F F F 11 220-239 195 195 195 338 338 338 F F F 12 240-259 195 195 195 338 338 338 F F F 13 260-279 195 195 195 338 338 338 F F F 14 280-299 192 192 192 333 333 333 F F F 15 300-319 187 187 187 323 323 323 F F F 16 320-339 194 194 194 337 337 337 F F F 17 340-359 190 190 190 329 329 329 F F F 18 360-379 192 192 192 331 331 331 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 191 191 191 330 330 330 F F F 21 420-439 187 187 187 323 323 323 F F F 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 192 192 192 332 332 332 F F F 26 520-539 194 194 194 336 336 336 F F F 27 540-559 194 194 194 336 336 336 F F F 28 560-579 194 194 194 336 336 336 F F F 29 580-599 194 194 194 336 336 336 F F F Max gen: 195, Max flush: 338 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.38935725Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:37.396803117Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.396853491Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.4038197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404090814Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40412921Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404138538Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404150557Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Crucible c4964321-38ac-4698-8c8f-ba3159d13214 has session id: 6bc06514-a43b-4c74-a4b9-e9eb9c421d53","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40451439Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Upstairs opts: Upstairs UUID: c4964321-38ac-4698-8c8f-ba3159d13214, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404578281Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Crucible stats registered with UUID: c4964321-38ac-4698-8c8f-ba3159d13214","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40459101Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:37.404602928Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404734341Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"1","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404766207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"0","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404777846Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"2","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404919697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404937605Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"c4964321-38ac-4698-8c8f-ba3159d13214 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404970921Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404980859Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"0","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.404990988Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"1","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405001047Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"2","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405011095Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"2","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405021594Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"0","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405060379Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"1","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"ds_connection connected from Ok(127.0.0.1:54385)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405084336Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"1","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"ds_connection connected from Ok(127.0.0.1:61917)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405221358Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"2","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"ds_connection connected from Ok(127.0.0.1:51812)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405234197Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"io task","client":"0","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405774747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"2","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405806303Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"1","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.405929867Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"0","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406181604Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[0]R flush_numbers[0..12]: [318, 329, 337, 335, 335, 332, 333, 337, 329, 335, 331, 338]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406208111Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[0]R generation[0..12]: [182, 190, 194, 193, 193, 192, 192, 194, 190, 193, 192, 195]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406219819Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406231348Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[1]R flush_numbers[0..12]: [318, 329, 337, 335, 335, 332, 333, 337, 329, 335, 331, 338]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406242516Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[1]R generation[0..12]: [182, 190, 194, 193, 193, 192, 192, 194, 190, 193, 192, 195]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406253475Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406264693Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[2]R flush_numbers[0..12]: [318, 329, 337, 335, 335, 332, 333, 337, 329, 335, 331, 338]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406275572Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[2]R generation[0..12]: [182, 190, 194, 193, 193, 192, 192, 194, 190, 193, 192, 195]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40628677Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406298519Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406308528Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406319826Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406329585Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406339384Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"":"downstairs","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406349552Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406358851Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"0","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40636899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"1","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406379019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"client":"2","":"downstairs","session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406389597Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406399356Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"c4964321-38ac-4698-8c8f-ba3159d13214 is now active with session: 6bc06514-a43b-4c74-a4b9-e9eb9c421d53","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406414554Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406423753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"session_id":"6bc06514-a43b-4c74-a4b9-e9eb9c421d53"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406433272Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Activated sub_volume c4964321-38ac-4698-8c8f-ba3159d13214","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40644161Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.406450029Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.40667899Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} 01/30 Flush 02/30 Write block 223 len 20480 data: 18 18 19 17 19 03/30 Read block 536 len 12288 04/30 Read block 99 len 12288 05/30 Read block 250 len 24576 06/30 Write block 411 len 40960 data: 6 6 7 9 9 13 17 17 15 14 07/30 Write block 334 len 4096 data: 17 08/30 Read block 403 len 8192 09/30 Read block 502 len 12288 10/30 Read block 436 len 4096 11/30 Write block 155 len 4096 data: 14 12/30 Read block 326 len 4096 13/30 Write block 348 len 20480 data: 14 16 20 17 16 14/30 Write block 316 len 32768 data: 14 15 10 6 9 9 8 9 15/30 Read block 145 len 32768 16/30 Write block 383 len 4096 data: 16 17/30 Write block 26 len 28672 data: 15 15 14 12 9 9 11 18/30 Read block 10 len 36864 19/30 Write block 110 len 32768 data: 14 12 10 14 12 14 11 14 20/30 Read block 370 len 16384 21/30 Read block 60 len 32768 22/30 Write block 246 len 8192 data: 15 16 23/30 Read block 383 len 24576 24/30 Write block 180 len 4096 data: 16 25/30 Read block 35 len 28672 26/30 Flush 27/30 Write block 228 len 28672 data: 16 11 9 8 7 9 8 28/30 Write block 133 len 40960 data: 19 18 16 17 17 16 17 14 11 14 29/30 Read block 579 len 28672 30/30 Write block 30 len 32768 data: 10 10 12 12 11 12 14 11 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 1 1 26 Done 5 28 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:37.415517137Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:37.415738848Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2027,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 190 196 196 329 340 340 F T T <--- 2 040-059 194 194 194 337 337 337 F F F 3 060-079 193 193 193 335 335 335 F F F 4 080-099 193 193 193 335 335 335 F F F 5 100-119 192 196 196 332 340 340 F F F <--- 6 120-139 192 192 192 333 333 333 F T T <--- 7 140-159 194 196 196 337 340 340 F T T <--- 8 160-179 190 190 190 329 329 329 F F F 9 180-199 193 196 196 335 340 340 F F F <--- 10 200-219 192 192 192 331 331 331 F F F 11 220-239 195 196 196 338 340 340 T T T <--- 12 240-259 195 196 196 338 340 340 F F F <--- 13 260-279 195 195 195 338 338 338 F F F 14 280-299 192 192 192 333 333 333 F F F 15 300-319 187 196 196 323 340 340 F F F <--- 16 320-339 194 196 196 337 340 340 F F F <--- 17 340-359 190 196 196 329 340 340 F F F <--- 18 360-379 192 192 192 331 331 331 F F F 19 380-399 191 196 196 330 340 340 F F F <--- 20 400-419 191 196 196 330 340 340 T F F <--- 21 420-439 187 196 196 323 340 340 T F F <--- 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 192 192 192 332 332 332 F F F 26 520-539 194 194 194 336 336 336 F F F 27 540-559 194 194 194 336 336 336 F F F 28 560-579 194 194 194 336 336 336 F F F 29 580-599 194 194 194 336 336 336 F F F Max gen: 196, Max flush: 340 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 197 197 197 341 341 341 F F F 2 040-059 194 194 194 337 337 337 F F F 3 060-079 193 193 193 335 335 335 F F F 4 080-099 193 193 193 335 335 335 F F F 5 100-119 196 196 196 340 340 340 F F F 6 120-139 197 197 197 341 341 341 F F F 7 140-159 197 197 197 341 341 341 F F F 8 160-179 190 190 190 329 329 329 F F F 9 180-199 196 196 196 340 340 340 F F F 10 200-219 192 192 192 331 331 331 F F F 11 220-239 197 197 197 341 341 341 F F F 12 240-259 196 196 196 340 340 340 F F F 13 260-279 195 195 195 338 338 338 F F F 14 280-299 192 192 192 333 333 333 F F F 15 300-319 196 196 196 340 340 340 F F F 16 320-339 196 196 196 340 340 340 F F F 17 340-359 196 196 196 340 340 340 F F F 18 360-379 192 192 192 331 331 331 F F F 19 380-399 196 196 196 340 340 340 F F F 20 400-419 196 196 196 340 340 340 F F F 21 420-439 196 196 196 340 340 340 F F F 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 192 192 192 332 332 332 F F F 26 520-539 194 194 194 336 336 336 F F F 27 540-559 194 194 194 336 336 336 F F F 28 560-579 194 194 194 336 336 336 F F F 29 580-599 194 194 194 336 336 336 F F F Max gen: 197, Max flush: 341 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.681673249Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:51.689032057Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.689080271Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.696374858Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.696671769Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"16c0b325dfca7b7496108538749b3fe72ec2272f\",\n git_commit_timestamp: \"2026-05-14T20:29:18.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.696706445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.696715724Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.696728672Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Crucible 77aab434-8fcc-4139-945f-4257ebb78d06 has session id: 565557c5-9219-4abf-ae27-ad19d46dea13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697153407Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Upstairs opts: Upstairs UUID: 77aab434-8fcc-4139-945f-4257ebb78d06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697184753Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Crucible stats registered with UUID: 77aab434-8fcc-4139-945f-4257ebb78d06","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697196661Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:51.69720679Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697342393Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"0","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.69736019Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"2","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697371249Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"1","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697534308Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697548296Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"77aab434-8fcc-4139-945f-4257ebb78d06 active request set","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697567603Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697577192Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"0","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697587651Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"1","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697598089Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"2","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697608708Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"2","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697619697Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"0","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697630555Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"1","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"ds_connection connected from Ok(127.0.0.1:36553)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697705756Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"2","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"ds_connection connected from Ok(127.0.0.1:52681)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.697733282Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"0","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"ds_connection connected from Ok(127.0.0.1:41436)","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.69774591Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"io task","client":"1","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698416524Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"1","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698546207Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"0","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.69859635Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"2","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698869745Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[0]R flush_numbers[0..12]: [318, 341, 337, 335, 335, 340, 341, 341, 329, 340, 331, 341]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698893592Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[0]R generation[0..12]: [182, 197, 194, 193, 193, 196, 197, 197, 190, 196, 192, 197]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.69890539Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698917259Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[1]R flush_numbers[0..12]: [318, 341, 337, 335, 335, 340, 341, 341, 329, 340, 331, 341]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698928747Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[1]R generation[0..12]: [182, 197, 194, 193, 193, 196, 197, 197, 190, 196, 192, 197]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698940156Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698952034Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[2]R flush_numbers[0..12]: [318, 341, 337, 335, 335, 340, 341, 341, 329, 340, 331, 341]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698963703Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[2]R generation[0..12]: [182, 197, 194, 193, 193, 196, 197, 197, 190, 196, 192, 197]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698975071Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.69898687Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.698997068Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699007707Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699018086Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699028244Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"":"downstairs","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699038813Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699048332Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"0","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.69906018Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"1","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699070489Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"client":"2","":"downstairs","session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699081018Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699090806Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"77aab434-8fcc-4139-945f-4257ebb78d06 is now active with session: 565557c5-9219-4abf-ae27-ad19d46dea13","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699101445Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699110874Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"session_id":"565557c5-9219-4abf-ae27-ad19d46dea13"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucibleDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","level":30,"time":"2026-05-14T21:09:51.699126932Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Activated sub_volume 77aab434-8fcc-4139-945f-4257ebb78d06","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699135841Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699144709Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.699409255Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} 01/30 Flush 02/30 Write block 430 len 12288 data: 11 14 14 03/30 Read block 274 len 24576 04/30 Write block 577 len 32768 data: 18 17 15 17 15 14 16 15 05/30 Write block 423 len 20480 data: 13 10 8 8 9 06/30 Read block 77 len 36864 07/30 Write block 515 len 4096 data: 13 08/30 Write block 536 len 20480 data: 21 23 24 22 24 09/30 Flush 10/30 Write block 46 len 16384 data: 18 24 22 20 11/30 Read block 58 len 36864 12/30 Read block 93 len 28672 13/30 Read block 384 len 36864 14/30 Read block 393 len 36864 15/30 Read block 416 len 32768 16/30 Read block 440 len 24576 17/30 Flush 18/30 Write block 429 len 32768 data: 10 12 15 15 16 19 19 21 19/30 Read block 468 len 16384 20/30 Write block 62 len 24576 data: 15 14 13 14 14 14 21/30 Read block 403 len 4096 22/30 Write block 579 len 12288 data: 16 18 16 23/30 Read block 231 len 16384 24/30 Write block 532 len 36864 data: 14 16 14 19 22 24 25 23 25 25/30 Read block 548 len 8192 26/30 Write block 270 len 32768 data: 18 21 19 19 20 20 19 15 27/30 Read block 282 len 40960 28/30 Write block 282 len 24576 data: 12 13 10 14 15 16 29/30 Read block 83 len 24576 30/30 Read block 225 len 32768 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 1 28 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-05-14T21:09:51.709676397Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-05-14T21:09:51.70988394Z","hostname":"w-01KRM3RCFBADMBR1YTAXV03MZQ","pid":2036,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 197 197 197 341 341 341 F F F 2 040-059 194 198 198 337 344 344 F F F <--- 3 060-079 193 193 193 335 335 335 F T T <--- 4 080-099 193 193 193 335 335 335 F F F 5 100-119 196 196 196 340 340 340 F F F 6 120-139 197 197 197 341 341 341 F F F 7 140-159 197 197 197 341 341 341 F F F 8 160-179 190 190 190 329 329 329 F F F 9 180-199 196 196 196 340 340 340 F F F 10 200-219 192 192 192 331 331 331 F F F 11 220-239 197 197 197 341 341 341 F F F 12 240-259 196 196 196 340 340 340 F F F 13 260-279 195 195 195 338 338 338 F T T <--- 14 280-299 192 192 192 333 333 333 F T T <--- 15 300-319 196 196 196 340 340 340 F F F 16 320-339 196 196 196 340 340 340 F F F 17 340-359 196 196 196 340 340 340 F F F 18 360-379 192 192 192 331 331 331 F F F 19 380-399 196 196 196 340 340 340 F F F 20 400-419 196 196 196 340 340 340 F F F 21 420-439 196 198 198 340 343 343 T T T <--- 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 192 198 198 332 343 343 F F F <--- 26 520-539 194 198 198 336 343 343 F T T <--- 27 540-559 194 198 198 336 343 343 F T T <--- 28 560-579 194 198 198 336 343 343 F T T <--- 29 580-599 194 198 198 336 343 343 F T T <--- Max gen: 198, Max flush: 344 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 318 318 318 F F F 1 020-039 197 197 197 341 341 341 F F F 2 040-059 198 198 198 344 344 344 F F F 3 060-079 199 199 199 345 345 345 F F F 4 080-099 193 193 193 335 335 335 F F F 5 100-119 196 196 196 340 340 340 F F F 6 120-139 197 197 197 341 341 341 F F F 7 140-159 197 197 197 341 341 341 F F F 8 160-179 190 190 190 329 329 329 F F F 9 180-199 196 196 196 340 340 340 F F F 10 200-219 192 192 192 331 331 331 F F F 11 220-239 197 197 197 341 341 341 F F F 12 240-259 196 196 196 340 340 340 F F F 13 260-279 199 199 199 345 345 345 F F F 14 280-299 199 199 199 345 345 345 F F F 15 300-319 196 196 196 340 340 340 F F F 16 320-339 196 196 196 340 340 340 F F F 17 340-359 196 196 196 340 340 340 F F F 18 360-379 192 192 192 331 331 331 F F F 19 380-399 196 196 196 340 340 340 F F F 20 400-419 196 196 196 340 340 340 F F F 21 420-439 199 199 199 345 345 345 F F F 22 440-459 192 192 192 331 331 331 F F F 23 460-479 181 181 181 316 316 316 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 198 198 198 343 343 343 F F F 26 520-539 199 199 199 345 345 345 F F F 27 540-559 199 199 199 345 345 345 F F F 28 560-579 199 199 199 345 345 345 F F F 29 580-599 199 199 199 345 345 345 F F F Max gen: 199, Max flush: 345 24:00 Test duration Test completed real 24:00.493810401 user 27.826920309 sys 15.785151170 trap 0.023723000 tflt 0.073451367 dflt 0.084542269 kflt 0.014285799 lock 32:55:22.665353964 slp 1:56:00.799822636 lat 2.928949764 stop 0.023045601