Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.19741167 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.012928341 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.012180606 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.319655242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.331753148Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.332884925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.33299868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.333008089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.333020017Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"Crucible 906fca69-15c7-40b8-ad6d-8720554e1989 has session id: 48d031a0-f3f8-4fe1-aa2e-73669a555fa6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.333769412Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Upstairs opts: Upstairs UUID: 906fca69-15c7-40b8-ad6d-8720554e1989, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.333837154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Crucible stats registered with UUID: 906fca69-15c7-40b8-ad6d-8720554e1989","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.333847702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:14.333887267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334348609Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334405472Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.33441589Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334651541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334686166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"906fca69-15c7-40b8-ad6d-8720554e1989 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334711643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334721312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.33473143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334741329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"2","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334770745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334781444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334791503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38519)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334848266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60981)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334873502Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45577)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.334949943Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.336337917Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.336366963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.33638924Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"2","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337010961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337091081Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.33710184Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337112848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337123057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337133026Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337143824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337153913Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337163912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337174711Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337183949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337193538Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337202777Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337211886Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"downstairs","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337222974Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337231553Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337241362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337251061Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"client":"2","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.33726106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337270128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"906fca69-15c7-40b8-ad6d-8720554e1989 is now active with session: 48d031a0-f3f8-4fe1-aa2e-73669a555fa6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337280117Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337288666Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337297725Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"Activated sub_volume 906fca69-15c7-40b8-ad6d-8720554e1989","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337395273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337440247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.337601526Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- ,"time":"2026-10-01T00:45:14.376931218Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 {States:"msg":" ActiveCLIENT: Tests done. All submitted work has been ACK'd Active" Active," v":0,JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY "name":"crucible","level":30 1004 Acked Read 256," Donetime": Done" Done false 2026-10-01T00:45:14.376971993Z"," 1005 Acked Read 88hostname" Done:" Done Done false w-01M3TENQCF8Q6JV57EJ5FJRMYJ"," 1006 Acked Read 256pid" Done: Done1139 Done false, "task":"crutest"} STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:14.377037645Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.378480216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.37852972Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.378544378Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.378564025Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.378595092Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.37861105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.378625818Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.378640846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.378656094Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.378671652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.37868616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"48d031a0-f3f8-4fe1-aa2e-73669a555fa6"} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.40336254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.411504118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.411835896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.411865352Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.411876361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.411887949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Crucible c6406c9b-0328-4512-8a97-438c18db4d48 has session id: 1110e315-3983-48d9-8a7e-5b36c6926d7e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412253233Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Upstairs opts: Upstairs UUID: c6406c9b-0328-4512-8a97-438c18db4d48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412320634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Crucible stats registered with UUID: c6406c9b-0328-4512-8a97-438c18db4d48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412332433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.412341782Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412495542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412512Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412522389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412666071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412682388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"c6406c9b-0328-4512-8a97-438c18db4d48 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412697996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412706865Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"0","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412718634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"1","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412728223Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"2","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412737981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.41274785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412757919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64072)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.41282916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34543)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412865495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45848)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.412876364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413388909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"2","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413412976Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"0","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413544779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"1","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413848511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413873747Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413884516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413895495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413907813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413917822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.41392854Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413938809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413948728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413959586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413969065Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413978704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413987843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.413997042Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"":"downstairs","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414021239Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414029958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"0","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414039446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"1","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414048835Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"client":"2","":"downstairs","session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414058464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414069433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"c6406c9b-0328-4512-8a97-438c18db4d48 is now active with session: 1110e315-3983-48d9-8a7e-5b36c6926d7e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414079191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.41408785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"session_id":"1110e315-3983-48d9-8a7e-5b36c6926d7e"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414096779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Activated sub_volume c6406c9b-0328-4512-8a97-438c18db4d48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414104638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-10-01T00:45:18.414117007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.414472231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"task":"crutest"} 01/30 Write block 539 len 8192 data: 2 2 02/30 Write block 475 len 4096 data: 2 03/30 Read block 430 len 28672 04/30 Flush 05/30 Write block 122 len 8192 data: 2 2 06/30 Write block 168 len 40960 data: 2 2 2 2 2 2 2 2 2 2 07/30 Read block 370 len 28672 08/30 Write block 573 len 24576 data: 2 2 2 2 2 2 09/30 Flush 10/30 Write block 277 len 28672 data: 2 2 2 2 2 2 2 11/30 Write block 74 len 24576 data: 2 2 2 2 2 2 12/30 Write block 245 len 4096 data: 2 13/30 Read block 499 len 20480 14/30 Write block 364 len 4096 data: 2 15/30 Write block 63 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Read block 272 len 24576 17/30 Write block 88 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Read block 364 len 28672 19/30 Read block 444 len 32768 20/30 Read block 261 len 20480 21/30 Write block 343 len 12288 data: 2 2 2 22/30 Flush 23/30 Write block 138 len 8192 data: 2 2 24/30 Read block 27 len 40960 25/30 Write block 421 len 4096 data: 2 26/30 Read block 130 len 16384 27/30 Write block 107 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Read block 491 len 24576 29/30 Write block 195 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Read block 36 len 32768 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 21 1 22 Done 21 0 20 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1008)) Some(JobId(1021)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:18.423449143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:18.42362649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 1 2 4 1 4 F F F <--- 4 080-099 2 1 2 4 1 4 F F F <--- 5 100-119 1 1 1 1 1 1 T F T <--- 6 120-139 2 2 2 3 3 3 T F T <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 1 1 1 1 1 1 T F T <--- 10 200-219 1 1 1 1 1 1 T F T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 1 2 4 1 4 F F F <--- 13 260-279 2 1 2 4 1 4 F F F <--- 14 280-299 2 1 2 4 1 4 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 1 2 4 1 4 F F F <--- 18 360-379 2 1 2 4 1 4 F F F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 T F T <--- 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 2 2 2 4 4 4 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 4 4 4 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 4 4 4 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 5 5 5 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.803699152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.811641835Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.811929139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.811968534Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.81199544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812008369Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Crucible a13f6701-0a34-4157-a196-91021ca7a98c has session id: 96afea98-3ec9-4a37-9fda-a7ecf9088a59","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812295842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Upstairs opts: Upstairs UUID: a13f6701-0a34-4157-a196-91021ca7a98c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812318709Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Crucible stats registered with UUID: a13f6701-0a34-4157-a196-91021ca7a98c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812331947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:32.812341106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812465331Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"0","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812482338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"1","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812492347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"2","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812635749Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812659636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"a13f6701-0a34-4157-a196-91021ca7a98c active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812669565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812678204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"0","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812687792Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"1","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812697201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"2","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.81270672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"0","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812716719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"2","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812726407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"1","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"ds_connection connected from Ok(127.0.0.1:63083)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812799058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"1","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"ds_connection connected from Ok(127.0.0.1:59261)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812829234Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"2","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"ds_connection connected from Ok(127.0.0.1:61599)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.812840383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"io task","client":"0","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813522566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"1","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813557322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"2","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813604166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"0","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.81388568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 4, 4, 5, 5, 1, 3, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813912987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 2, 2, 3, 3, 1, 2, 3, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813923646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813934664Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 4, 4, 5, 5, 1, 3, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813944843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 2, 2, 3, 3, 1, 2, 3, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813954802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.81396558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 4, 4, 5, 5, 1, 3, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813975739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 2, 2, 3, 3, 1, 2, 3, 3, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813985688Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.813996406Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814005785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814015394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814026113Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814035451Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"":"downstairs","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.81404509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814053799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"0","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814063148Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"1","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814072767Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"client":"2","":"downstairs","session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814082495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814091544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"a13f6701-0a34-4157-a196-91021ca7a98c is now active with session: 96afea98-3ec9-4a37-9fda-a7ecf9088a59","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814101413Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814110002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"session_id":"96afea98-3ec9-4a37-9fda-a7ecf9088a59"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814118991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Activated sub_volume a13f6701-0a34-4157-a196-91021ca7a98c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.81412702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814135019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.814450069Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"task":"crutest"} 01/30 Write block 310 len 12288 data: 2 2 2 02/30 Read block 241 len 8192 03/30 Write block 400 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 365 len 4096 05/30 Read block 507 len 40960 06/30 Read block 89 len 4096 07/30 Read block 98 len 12288 08/30 Flush 09/30 Write block 539 len 32768 data: 3 3 2 2 2 2 2 2 10/30 Read block 244 len 4096 11/30 Read block 16 len 12288 12/30 Write block 71 len 36864 data: 3 3 2 3 3 3 3 3 3 13/30 Read block 266 len 36864 14/30 Read block 435 len 24576 15/30 Read block 218 len 36864 16/30 Read block 348 len 8192 17/30 Read block 476 len 28672 18/30 Flush 19/30 Write block 563 len 4096 data: 2 20/30 Write block 95 len 16384 data: 3 3 2 2 21/30 Write block 352 len 36864 data: 2 2 2 2 2 2 2 2 2 22/30 Flush 23/30 Write block 75 len 24576 data: 4 4 4 4 4 2 24/30 Flush 25/30 Write block 550 len 8192 data: 2 2 26/30 Read block 243 len 24576 27/30 Read block 517 len 12288 28/30 Read block 278 len 16384 29/30 Write block 208 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Write block 514 len 32768 data: 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 2 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:32.822846494Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:32.822991385Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1150,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 4 2 4 9 4 9 F F F <--- 4 080-099 4 2 4 9 4 9 F F F <--- 5 100-119 3 3 3 5 5 5 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 3 3 3 5 5 5 T F T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 4 4 4 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 4 1 4 6 1 6 F F F <--- 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 2 4 8 4 8 F F F <--- 18 360-379 4 2 4 8 4 8 F F F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 1 4 6 1 6 F F F <--- 21 420-439 3 3 3 5 5 5 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 T F T <--- 26 520-539 4 2 4 7 2 7 T F T <--- 27 540-559 4 2 4 7 2 7 T F T <--- 28 560-579 4 2 4 8 3 8 F F F <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 9 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 4 4 4 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 4 4 4 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 4 4 4 8 8 8 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 6 6 6 F F F 21 420-439 3 3 3 5 5 5 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 5 5 5 10 10 10 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 4 4 4 8 8 8 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.066111356Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:47.073143174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.073199047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.079985266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080237074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080260021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.08026842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080280109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Crucible d85164e0-69d8-4a21-a099-f195fd4bbe84 has session id: 587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080653331Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Upstairs opts: Upstairs UUID: d85164e0-69d8-4a21-a099-f195fd4bbe84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080681728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Crucible stats registered with UUID: d85164e0-69d8-4a21-a099-f195fd4bbe84","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080692607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:47.080701625Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080840938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080871804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.080882762Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081022425Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081038983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"d85164e0-69d8-4a21-a099-f195fd4bbe84 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081054821Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081063629Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"0","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081075168Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"1","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081084947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"2","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081094446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081104394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081114133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42005)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081165057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52387)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081175885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47634)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081186054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081823543Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"1","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.08184539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"2","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.081856909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"0","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082191416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 9, 9, 5, 5, 1, 3, 5, 10, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082220083Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 4, 4, 3, 3, 1, 2, 3, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082230931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.08224453Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 9, 9, 5, 5, 1, 3, 5, 10, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082254908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 4, 4, 3, 3, 1, 2, 3, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082265037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082275866Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 9, 9, 5, 5, 1, 3, 5, 10, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082286004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 4, 4, 3, 3, 1, 2, 3, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082296233Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082307232Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082316511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082326239Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082335638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082345007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"":"downstairs","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082354846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082363715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"0","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082373333Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"1","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082383062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"client":"2","":"downstairs","session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082394121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.08240317Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"d85164e0-69d8-4a21-a099-f195fd4bbe84 is now active with session: 587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082413208Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082421777Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"session_id":"587e68d1-78f0-4a85-a0c0-9d2c2ad54e5b"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082430726Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Activated sub_volume d85164e0-69d8-4a21-a099-f195fd4bbe84","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082438715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082446844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.082748396Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} 01/30 Write block 84 len 28672 data: 2 2 2 2 3 3 3 02/30 Read block 459 len 4096 03/30 Read block 69 len 8192 04/30 Write block 421 len 24576 data: 3 2 2 2 2 2 05/30 Write block 446 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Write block 276 len 32768 data: 2 3 3 3 3 3 3 3 07/30 Write block 437 len 16384 data: 2 2 2 2 08/30 Read block 549 len 8192 09/30 Write block 228 len 24576 data: 2 2 2 2 2 2 10/30 Read block 54 len 16384 11/30 Read block 403 len 28672 12/30 Flush 13/30 Write block 259 len 16384 data: 2 2 2 2 14/30 Write block 83 len 32768 data: 2 3 3 3 3 4 4 4 15/30 Read block 526 len 20480 16/30 Write block 401 len 24576 data: 3 3 3 3 3 3 17/30 Write block 293 len 20480 data: 2 2 2 2 2 18/30 Write block 485 len 12288 data: 2 2 2 19/30 Write block 327 len 16384 data: 2 2 2 2 20/30 Read block 485 len 28672 21/30 Read block 121 len 40960 22/30 Read block 577 len 32768 23/30 Write block 33 len 24576 data: 2 2 2 2 2 2 24/30 Write block 263 len 24576 data: 2 2 2 2 2 2 25/30 Write block 375 len 4096 data: 2 26/30 Write block 512 len 20480 data: 2 2 3 3 3 27/30 Write block 46 len 28672 data: 2 2 2 2 2 2 2 28/30 Read block 355 len 36864 29/30 Read block 484 len 8192 30/30 Read block 548 len 24576 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:45:47.090458418Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:45:47.090632776Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1159,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F T T <--- 2 040-059 1 1 1 1 1 1 F T T <--- 3 060-079 4 4 4 9 9 9 F F F 4 080-099 4 6 6 9 11 11 T T T <--- 5 100-119 3 3 3 5 5 5 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 1 6 6 1 11 11 F F F <--- 12 240-259 2 2 2 4 4 4 F T T <--- 13 260-279 2 6 6 4 11 11 F T T <--- 14 280-299 2 6 6 4 11 11 F T T <--- 15 300-319 4 4 4 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F T T <--- 17 340-359 4 4 4 8 8 8 F F F 18 360-379 4 4 4 8 8 8 F T T <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 6 6 6 F T T <--- 21 420-439 3 6 6 5 11 11 F F F <--- 22 440-459 1 6 6 1 11 11 F F F <--- 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F T T <--- 25 500-519 5 5 5 10 10 10 F T T <--- 26 520-539 5 5 5 10 10 10 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 4 4 4 8 8 8 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 11 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 7 7 7 12 12 12 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 7 7 7 12 12 12 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 4 4 4 6 6 6 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 7 7 7 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 7 7 7 12 12 12 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 7 7 7 12 12 12 F F F 25 500-519 7 7 7 12 12 12 F F F 26 520-539 5 5 5 10 10 10 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 4 4 4 8 8 8 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.340725742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:01.347702727Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.347736933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.354549859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.354810626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.354845601Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.35485493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.354866759Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Crucible d1e689f5-21f4-4667-b07a-9294e07d9134 has session id: 93d45603-f25c-4603-9181-b3641a1ef5e2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355245361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Upstairs opts: Upstairs UUID: d1e689f5-21f4-4667-b07a-9294e07d9134, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355287125Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Crucible stats registered with UUID: d1e689f5-21f4-4667-b07a-9294e07d9134","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355297664Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:01.355307573Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355457104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"0","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355474002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"1","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.35548407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"2","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355627612Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355650969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"d1e689f5-21f4-4667-b07a-9294e07d9134 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355667187Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355676046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"0","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355685615Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"1","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355695273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"2","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355704892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"2","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355714761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"0","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.3557246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"1","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50063)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355793231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"2","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41778)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355818528Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"1","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63612)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.355829766Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"io task","client":"0","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356470695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"1","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356633005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"2","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356663301Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"0","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356915559Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[0]R flush_numbers[0..12]: [1, 12, 12, 9, 12, 5, 5, 1, 3, 5, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356934566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[0]R generation[0..12]: [1, 7, 7, 4, 7, 3, 3, 1, 2, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356945015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356956044Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[1]R flush_numbers[0..12]: [1, 12, 12, 9, 12, 5, 5, 1, 3, 5, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356966182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[1]R generation[0..12]: [1, 7, 7, 4, 7, 3, 3, 1, 2, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356976191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.35698708Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[2]R flush_numbers[0..12]: [1, 12, 12, 9, 12, 5, 5, 1, 3, 5, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.356997478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[2]R generation[0..12]: [1, 7, 7, 4, 7, 3, 3, 1, 2, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357010697Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357021605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357030974Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357040663Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357049822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.35705909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"":"downstairs","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357068739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357077488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"0","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357086977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"1","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357096516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"client":"2","":"downstairs","session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357106215Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357115183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"d1e689f5-21f4-4667-b07a-9294e07d9134 is now active with session: 93d45603-f25c-4603-9181-b3641a1ef5e2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357124802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357133481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"session_id":"93d45603-f25c-4603-9181-b3641a1ef5e2"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.35714246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Activated sub_volume d1e689f5-21f4-4667-b07a-9294e07d9134","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357150449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357160038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.357474338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} 01/30 Flush 02/30 Write block 131 len 36864 data: 2 2 2 2 2 2 2 3 3 03/30 Flush 04/30 Write block 525 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Write block 85 len 28672 data: 4 4 4 5 5 5 3 06/30 Read block 339 len 12288 07/30 Write block 167 len 36864 data: 2 3 3 3 3 3 3 3 3 08/30 Write block 400 len 20480 data: 3 4 4 4 4 09/30 Write block 352 len 20480 data: 3 3 3 3 3 10/30 Write block 127 len 40960 data: 2 2 2 2 3 3 3 3 3 3 11/30 Write block 175 len 36864 data: 4 3 3 2 2 2 2 2 2 12/30 Write block 500 len 20480 data: 2 2 2 2 2 13/30 Read block 5 len 28672 14/30 Read block 385 len 40960 15/30 Write block 571 len 16384 data: 2 2 3 3 16/30 Read block 597 len 4096 17/30 Read block 267 len 20480 18/30 Read block 221 len 4096 19/30 Read block 308 len 20480 20/30 Read block 355 len 24576 21/30 Read block 96 len 24576 22/30 Write block 138 len 28672 data: 4 4 2 2 2 2 2 23/30 Read block 26 len 28672 24/30 Write block 558 len 12288 data: 2 2 2 25/30 Write block 135 len 32768 data: 4 4 3 5 5 3 3 3 26/30 Write block 105 len 24576 data: 2 2 3 3 3 3 27/30 Write block 491 len 12288 data: 2 2 2 28/30 Read block 264 len 16384 29/30 Read block 332 len 20480 30/30 Read block 272 len 8192 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:01.366181464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:01.366351302Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1168,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 7 7 7 12 12 12 F T T <--- 5 100-119 3 3 3 5 5 5 F T T <--- 6 120-139 3 8 8 5 14 14 F T T <--- 7 140-159 1 1 1 1 1 1 F T T <--- 8 160-179 2 2 2 3 3 3 F T T <--- 9 180-199 3 3 3 5 5 5 F T T <--- 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 7 7 7 12 12 12 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 4 4 4 6 6 6 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 4 4 4 8 8 8 F T T <--- 18 360-379 7 7 7 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 7 7 7 12 12 12 F T T <--- 21 420-439 6 6 6 11 11 11 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 7 7 7 12 12 12 F T T <--- 25 500-519 7 7 7 12 12 12 F T T <--- 26 520-539 5 5 5 10 10 10 F T T <--- 27 540-559 5 5 5 10 10 10 F T T <--- 28 560-579 4 4 4 8 8 8 F T T <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 9 9 9 15 15 15 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 9 9 9 15 15 15 F F F 8 160-179 9 9 9 15 15 15 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 7 7 7 12 12 12 F F F 14 280-299 7 7 7 12 12 12 F F F 15 300-319 4 4 4 6 6 6 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 9 9 9 15 15 15 F F F 18 360-379 7 7 7 12 12 12 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 9 9 9 15 15 15 F F F 27 540-559 9 9 9 15 15 15 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.618382143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.626398626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.626675641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.626710026Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.626719425Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.626731114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Crucible 927b3528-e537-4c8c-9fe6-5f0c2a52f50e has session id: d43234bf-e113-4731-bd80-1213adbb37cf","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627087719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Upstairs opts: Upstairs UUID: 927b3528-e537-4c8c-9fe6-5f0c2a52f50e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627115315Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Crucible stats registered with UUID: 927b3528-e537-4c8c-9fe6-5f0c2a52f50e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627126344Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:15.627135482Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627278424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627294452Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627306711Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627438304Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627454682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"927b3528-e537-4c8c-9fe6-5f0c2a52f50e active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627486768Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627496027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"0","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627505646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"1","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627515174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"2","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627524643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627534592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627544331Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:40374)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627616671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:64401)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.627649607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54274)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.62771051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628294835Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"0","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628313563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"1","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628381734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"2","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628673667Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[0]R flush_numbers[0..12]: [1, 12, 12, 9, 15, 15, 15, 15, 15, 15, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628709153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[0]R generation[0..12]: [1, 7, 7, 4, 9, 9, 9, 9, 9, 9, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628719692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.62873058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[1]R flush_numbers[0..12]: [1, 12, 12, 9, 15, 15, 15, 15, 15, 15, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628740869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[1]R generation[0..12]: [1, 7, 7, 4, 9, 9, 9, 9, 9, 9, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628750998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628761686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[2]R flush_numbers[0..12]: [1, 12, 12, 9, 15, 15, 15, 15, 15, 15, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628771965Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[2]R generation[0..12]: [1, 7, 7, 4, 9, 9, 9, 9, 9, 9, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628781924Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628792772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628802321Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.62881198Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628821159Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628830527Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"":"downstairs","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628841796Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628850465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"0","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628860334Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"1","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628870122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"client":"2","":"downstairs","session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628880151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.62888934Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"927b3528-e537-4c8c-9fe6-5f0c2a52f50e is now active with session: d43234bf-e113-4731-bd80-1213adbb37cf","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628899219Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628908018Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"session_id":"d43234bf-e113-4731-bd80-1213adbb37cf"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628917067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Activated sub_volume 927b3528-e537-4c8c-9fe6-5f0c2a52f50e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628925005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.628933154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.629180463Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"task":"crutest"} 01/30 Flush 02/30 Write block 285 len 24576 data: 2 2 2 2 2 2 03/30 Flush 04/30 Write block 525 len 36864 data: 3 3 3 3 3 3 3 3 3 05/30 Write block 534 len 32768 data: 2 2 2 2 2 4 4 3 06/30 Flush 07/30 Write block 357 len 20480 data: 3 3 3 3 2 08/30 Write block 517 len 20480 data: 3 3 3 3 3 09/30 Write block 248 len 20480 data: 2 2 2 2 2 10/30 Read block 469 len 4096 11/30 Read block 309 len 20480 12/30 Read block 500 len 24576 13/30 Read block 411 len 40960 14/30 Read block 356 len 12288 15/30 Write block 591 len 24576 data: 2 2 2 2 2 2 16/30 Write block 251 len 20480 data: 3 3 2 2 2 17/30 Read block 257 len 8192 18/30 Write block 146 len 20480 data: 2 2 2 2 2 19/30 Read block 451 len 4096 20/30 Read block 483 len 36864 21/30 Read block 189 len 20480 22/30 Read block 89 len 8192 23/30 Read block 52 len 8192 24/30 Write block 445 len 12288 data: 2 3 3 25/30 Write block 71 len 20480 data: 4 4 3 4 5 26/30 Read block 48 len 20480 27/30 Write block 303 len 20480 data: 2 2 2 2 2 28/30 Read block 129 len 28672 29/30 Write block 40 len 32768 data: 2 2 2 2 2 2 3 3 30/30 Write block 161 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 27 27 1 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:15.637110117Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:15.637269017Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1177,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 T T F <--- 3 060-079 4 4 4 9 9 9 T T F <--- 4 080-099 9 9 9 15 15 15 F F F 5 100-119 9 9 9 15 15 15 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 9 9 9 15 15 15 T T F <--- 8 160-179 9 9 9 15 15 15 T T F <--- 9 180-199 9 9 9 15 15 15 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 7 7 7 12 12 12 T T F <--- 13 260-279 7 7 7 12 12 12 F F F 14 280-299 10 10 7 17 17 12 F F T <--- 15 300-319 4 4 4 6 6 6 T T F <--- 16 320-339 7 7 7 12 12 12 F F F 17 340-359 9 9 9 15 15 15 T T F <--- 18 360-379 7 7 7 12 12 12 T T F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 6 6 6 11 11 11 T T F <--- 23 460-479 2 2 2 2 2 2 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 9 9 9 15 15 15 T T F <--- 26 520-539 10 10 9 18 18 15 T T F <--- 27 540-559 10 10 9 18 18 15 F F F <--- 28 560-579 9 9 9 15 15 15 F F F 29 580-599 1 1 1 1 1 1 T T F <--- Max gen: 10, Max flush: 18 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 9 9 9 15 15 15 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 11 11 11 19 19 19 F F F 8 160-179 11 11 11 19 19 19 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 7 7 7 12 12 12 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 11 11 11 19 19 19 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 11 11 11 19 19 19 F F F 18 360-379 11 11 11 19 19 19 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 11 11 11 19 19 19 F F F 26 520-539 11 11 11 19 19 19 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 11 11 11 19 19 19 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.889504371Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.897426967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.897726499Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.897752295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.897760964Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.897775392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Crucible fb09723d-822f-473c-ab0a-aca706b85562 has session id: 44a3f01f-cb63-42bb-9f61-259eb65fd87e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898104191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Upstairs opts: Upstairs UUID: fb09723d-822f-473c-ab0a-aca706b85562, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898133217Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Crucible stats registered with UUID: fb09723d-822f-473c-ab0a-aca706b85562","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898144176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:29.898153255Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898286108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898301936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898311844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898467825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898485102Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"fb09723d-822f-473c-ab0a-aca706b85562 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898508839Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898518088Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"0","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898527857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"1","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898537446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"2","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898547005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898558763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898568622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57133)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898612046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"ds_connection connected from Ok(127.0.0.1:32806)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898622845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34260)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.898633024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899280462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"1","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899315597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"0","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899337024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"2","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899667672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[0]R flush_numbers[0..12]: [1, 12, 19, 19, 15, 15, 15, 19, 19, 15, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899697659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[0]R generation[0..12]: [1, 7, 11, 11, 9, 9, 9, 11, 11, 9, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899708487Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899719516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[1]R flush_numbers[0..12]: [1, 12, 19, 19, 15, 15, 15, 19, 19, 15, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899729935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[1]R generation[0..12]: [1, 7, 11, 11, 9, 9, 9, 11, 11, 9, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899739923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899753322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[2]R flush_numbers[0..12]: [1, 12, 19, 19, 15, 15, 15, 19, 19, 15, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.89976373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[2]R generation[0..12]: [1, 7, 11, 11, 9, 9, 9, 11, 11, 9, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899774019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899784988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899794436Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899804035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899813254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899822543Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"":"downstairs","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899832172Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899840711Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"0","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899850289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"1","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899859838Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"client":"2","":"downstairs","session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899869757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899878946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"fb09723d-822f-473c-ab0a-aca706b85562 is now active with session: 44a3f01f-cb63-42bb-9f61-259eb65fd87e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899888834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899897433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"session_id":"44a3f01f-cb63-42bb-9f61-259eb65fd87e"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899908242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Activated sub_volume fb09723d-822f-473c-ab0a-aca706b85562","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.899916091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.89992415Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.900232401Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"task":"crutest"} 01/30 Write block 256 len 20480 data: 2 2 2 3 3 02/30 Write block 278 len 36864 data: 4 4 4 4 4 4 2 3 3 03/30 Flush 04/30 Write block 532 len 32768 data: 4 4 3 3 3 3 3 5 05/30 Read block 506 len 36864 06/30 Read block 210 len 20480 07/30 Read block 356 len 8192 08/30 Write block 253 len 28672 data: 3 3 3 3 3 3 4 09/30 Read block 544 len 36864 10/30 Flush 11/30 Write block 156 len 12288 data: 2 2 2 12/30 Write block 389 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Write block 163 len 36864 data: 3 3 2 2 3 4 4 4 4 14/30 Read block 435 len 12288 15/30 Read block 425 len 28672 16/30 Flush 17/30 Write block 101 len 32768 data: 2 2 2 2 3 3 4 4 18/30 Read block 241 len 8192 19/30 Flush 20/30 Write block 392 len 16384 data: 3 3 3 3 21/30 Read block 191 len 20480 22/30 Flush 23/30 Write block 11 len 32768 data: 2 2 2 2 2 2 2 2 24/30 Write block 505 len 4096 data: 2 25/30 Write block 368 len 28672 data: 2 2 2 2 2 2 2 26/30 Write block 252 len 8192 data: 4 4 27/30 Write block 298 len 20480 data: 2 2 2 2 2 28/30 Write block 242 len 8192 data: 2 2 29/30 Write block 108 len 24576 data: 5 4 4 3 3 3 30/30 Write block 183 len 16384 data: 3 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 8 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 7 Done Sent Sent false 1025 Acked Write 2 Sent Sent Sent false 1026 Acked Write 5 Sent Sent Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 5 6 24 35 Done 22 21 3 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:29.908117341Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:29.908288219Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1186,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T F <--- 1 020-039 7 7 7 12 12 12 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 12 12 9 23 23 15 T T F <--- 6 120-139 9 9 9 15 15 15 F F F 7 140-159 12 12 11 22 22 19 F F F <--- 8 160-179 12 12 11 22 22 19 F F F <--- 9 180-199 9 9 9 15 15 15 T T F <--- 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 12 12 12 21 21 20 T T F <--- 13 260-279 12 12 12 20 20 20 F F F 14 280-299 12 12 12 20 20 20 T T F <--- 15 300-319 11 11 11 19 19 19 T T F <--- 16 320-339 7 7 7 12 12 12 F F F 17 340-359 11 11 11 19 19 19 F F F 18 360-379 11 11 11 19 19 19 T T F <--- 19 380-399 12 12 1 24 24 1 F F F <--- 20 400-419 9 9 9 15 15 15 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 11 11 11 19 19 19 T T F <--- 26 520-539 12 12 11 21 21 19 F F T <--- 27 540-559 10 10 10 18 18 18 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 11 11 11 19 19 19 F F F Max gen: 12, Max flush: 24 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 13 13 13 25 25 25 F F F 6 120-139 9 9 9 15 15 15 F F F 7 140-159 12 12 12 22 22 22 F F F 8 160-179 12 12 12 22 22 22 F F F 9 180-199 13 13 13 25 25 25 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 13 13 13 25 25 25 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 13 13 13 25 25 25 F F F 15 300-319 13 13 13 25 25 25 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 11 11 11 19 19 19 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 12 12 12 24 24 24 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 6 6 6 11 11 11 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 9 9 9 15 15 15 F F F 29 580-599 11 11 11 19 19 19 F F F Max gen: 13, Max flush: 25 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.15889657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.166908724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167200247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167237982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167247071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.16725889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Crucible daccbcf4-e717-458e-946f-0076b0ef5378 has session id: 77b37eb9-a09c-4683-8a71-ef263f12a763","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167534955Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Upstairs opts: Upstairs UUID: daccbcf4-e717-458e-946f-0076b0ef5378, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167599657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Crucible stats registered with UUID: daccbcf4-e717-458e-946f-0076b0ef5378","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167610075Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:44.167619164Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167775234Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167794072Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167804201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.167996876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168014284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"daccbcf4-e717-458e-946f-0076b0ef5378 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168029912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168038471Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"0","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.16804802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"1","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168057579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"2","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168067047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168076786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168086605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"ds_connection connected from Ok(127.0.0.1:51346)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168155996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"ds_connection connected from Ok(127.0.0.1:64744)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168180743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"ds_connection connected from Ok(127.0.0.1:35365)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168301068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168869536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"1","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168930048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"0","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.168961254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"2","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169269695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[0]R flush_numbers[0..12]: [25, 12, 19, 19, 15, 25, 15, 22, 22, 25, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169288492Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[0]R generation[0..12]: [13, 7, 11, 11, 9, 13, 9, 12, 12, 13, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169299201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.16931027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[1]R flush_numbers[0..12]: [25, 12, 19, 19, 15, 25, 15, 22, 22, 25, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169320828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[1]R generation[0..12]: [13, 7, 11, 11, 9, 13, 9, 12, 12, 13, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169330987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169341786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[2]R flush_numbers[0..12]: [25, 12, 19, 19, 15, 25, 15, 22, 22, 25, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169352134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[2]R generation[0..12]: [13, 7, 11, 11, 9, 13, 9, 12, 12, 13, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169362343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169373222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.16938269Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169392419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169401678Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169412787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"":"downstairs","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169422225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169430874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"0","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169440453Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"1","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169450062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"client":"2","":"downstairs","session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169459641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.16946876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"daccbcf4-e717-458e-946f-0076b0ef5378 is now active with session: 77b37eb9-a09c-4683-8a71-ef263f12a763","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169478608Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169487347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"session_id":"77b37eb9-a09c-4683-8a71-ef263f12a763"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169496336Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Activated sub_volume daccbcf4-e717-458e-946f-0076b0ef5378","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169504185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169512204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.169802567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"task":"crutest"} 01/30 Write block 183 len 8192 data: 4 3 02/30 Read block 364 len 28672 03/30 Read block 204 len 24576 04/30 Read block 460 len 8192 05/30 Write block 427 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Read block 253 len 24576 07/30 Read block 559 len 28672 08/30 Flush 09/30 Write block 106 len 16384 data: 4 5 6 5 10/30 Read block 293 len 12288 11/30 Write block 156 len 36864 data: 3 3 3 2 2 3 3 4 4 12/30 Read block 466 len 32768 13/30 Write block 423 len 20480 data: 3 3 3 3 3 14/30 Write block 138 len 32768 data: 6 6 4 4 4 3 3 2 15/30 Read block 409 len 12288 16/30 Write block 389 len 24576 data: 3 3 3 4 4 4 17/30 Read block 589 len 12288 18/30 Read block 440 len 8192 19/30 Flush 20/30 Write block 579 len 40960 data: 2 2 2 2 2 2 2 2 2 2 21/30 Read block 324 len 16384 22/30 Write block 468 len 40960 data: 2 2 2 2 2 2 2 3 2 2 23/30 Read block 139 len 20480 24/30 Read block 511 len 24576 25/30 Read block 248 len 20480 26/30 Flush 27/30 Write block 357 len 12288 data: 4 4 4 28/30 Read block 57 len 32768 29/30 Write block 236 len 12288 data: 2 2 2 30/30 Read block 304 len 24576 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 2 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:44.179700242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:44.179865351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1195,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 14 14 13 27 27 25 F F F <--- 6 120-139 14 14 9 27 27 15 F F F <--- 7 140-159 14 14 12 27 27 22 F F F <--- 8 160-179 14 14 12 27 27 22 F F F <--- 9 180-199 14 14 13 26 26 25 F F T <--- 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 T T F <--- 12 240-259 13 13 13 25 25 25 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 13 13 13 25 25 25 F F F 15 300-319 13 13 13 25 25 25 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 11 11 11 19 19 19 T T F <--- 18 360-379 13 13 13 25 25 25 F F F 19 380-399 14 14 12 27 27 24 F F F <--- 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 6 27 27 11 F F F <--- 22 440-459 11 11 11 19 19 19 F F F 23 460-479 14 14 2 28 28 2 F F F <--- 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 14 14 9 28 28 15 F F F <--- 29 580-599 14 14 11 28 28 19 F F F <--- Max gen: 14, Max flush: 28 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 14 14 14 27 27 27 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 15 15 15 29 29 29 F F F 12 240-259 13 13 13 25 25 25 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 13 13 13 25 25 25 F F F 15 300-319 13 13 13 25 25 25 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 13 13 13 25 25 25 F F F 19 380-399 14 14 14 27 27 27 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 11 11 11 19 19 19 F F F 23 460-479 14 14 14 28 28 28 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 14 14 14 28 28 28 F F F 29 580-599 14 14 14 28 28 28 F F F Max gen: 15, Max flush: 29 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.428038801Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.435960146Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436238131Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436266047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436277736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436289235Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Crucible e715fa93-50b5-4bc8-9611-840e962f0e6b has session id: b9d1387f-5a87-4627-a644-2bbace4a4300","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436573299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Upstairs opts: Upstairs UUID: e715fa93-50b5-4bc8-9611-840e962f0e6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436602325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Crucible stats registered with UUID: e715fa93-50b5-4bc8-9611-840e962f0e6b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436613513Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:58.436622602Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436765234Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436783472Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436793511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436938822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.4369555Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"e715fa93-50b5-4bc8-9611-840e962f0e6b active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436973038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436981817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"0","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.436991405Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"1","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437000804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"2","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437012523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437022402Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.43703231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"ds_connection connected from Ok(127.0.0.1:62535)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437094002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"ds_connection connected from Ok(127.0.0.1:56967)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437105051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"ds_connection connected from Ok(127.0.0.1:55494)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.43711544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437750479Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"1","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437782255Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"2","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.437794044Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"0","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438108684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[0]R flush_numbers[0..12]: [25, 12, 19, 19, 15, 27, 27, 27, 27, 26, 10, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.43813678Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[0]R generation[0..12]: [13, 7, 11, 11, 9, 14, 14, 14, 14, 14, 5, 15]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438147879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438158977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[1]R flush_numbers[0..12]: [25, 12, 19, 19, 15, 27, 27, 27, 27, 26, 10, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438169476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[1]R generation[0..12]: [13, 7, 11, 11, 9, 14, 14, 14, 14, 14, 5, 15]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438182224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438193153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[2]R flush_numbers[0..12]: [25, 12, 19, 19, 15, 27, 27, 27, 27, 26, 10, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438203392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[2]R generation[0..12]: [13, 7, 11, 11, 9, 14, 14, 14, 14, 14, 5, 15]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.43821368Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438224509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438233998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438243667Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438252786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438262154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"":"downstairs","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438272023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438280772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"0","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438290591Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"1","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438300289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"client":"2","":"downstairs","session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438310258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438319367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"e715fa93-50b5-4bc8-9611-840e962f0e6b is now active with session: b9d1387f-5a87-4627-a644-2bbace4a4300","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438329256Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438339155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"session_id":"b9d1387f-5a87-4627-a644-2bbace4a4300"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438348163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Activated sub_volume e715fa93-50b5-4bc8-9611-840e962f0e6b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438356192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438364311Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.438638477Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"task":"crutest"} 01/30 Write block 190 len 20480 data: 2 2 2 2 2 02/30 Read block 427 len 8192 03/30 Read block 539 len 8192 04/30 Read block 77 len 12288 05/30 Read block 18 len 32768 06/30 Write block 287 len 8192 data: 3 3 07/30 Read block 423 len 40960 08/30 Read block 266 len 28672 09/30 Read block 348 len 24576 10/30 Read block 479 len 16384 11/30 Read block 456 len 32768 12/30 Write block 162 len 4096 data: 4 13/30 Write block 29 len 16384 data: 2 2 2 2 14/30 Read block 170 len 8192 15/30 Read block 526 len 8192 16/30 Write block 452 len 32768 data: 3 3 3 2 2 2 2 2 17/30 Read block 86 len 12288 18/30 Read block 437 len 36864 19/30 Write block 369 len 40960 data: 3 3 3 3 3 3 3 2 2 2 20/30 Write block 463 len 40960 data: 2 2 2 2 2 3 3 3 3 3 21/30 Write block 363 len 24576 data: 2 3 2 2 2 3 22/30 Flush 23/30 Write block 457 len 40960 data: 3 3 3 2 2 2 3 3 3 3 24/30 Write block 216 len 36864 data: 3 2 2 2 2 2 2 2 2 25/30 Read block 330 len 24576 26/30 Read block 413 len 12288 27/30 Write block 275 len 16384 data: 2 3 4 5 28/30 Write block 98 len 16384 data: 3 2 2 3 29/30 Read block 348 len 40960 30/30 Write block 183 len 16384 data: 5 4 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 2 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:46:58.447214199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:46:58.447383018Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1204,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 16 7 16 30 12 30 F F F <--- 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 9 9 9 15 15 15 T F T <--- 5 100-119 14 14 14 27 27 27 T F T <--- 6 120-139 14 14 14 27 27 27 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 16 14 16 30 27 30 F F F <--- 9 180-199 16 14 16 30 26 30 T F T <--- 10 200-219 5 5 5 10 10 10 T F T <--- 11 220-239 15 15 15 29 29 29 T F T <--- 12 240-259 13 13 13 25 25 25 F F F 13 260-279 12 12 12 20 20 20 T F T <--- 14 280-299 16 13 16 30 25 30 F F F <--- 15 300-319 13 13 13 25 25 25 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 16 13 16 30 25 30 F F F <--- 19 380-399 14 14 14 27 27 27 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 16 11 16 30 19 30 T F T <--- 23 460-479 16 14 16 30 28 30 T F T <--- 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 14 14 14 28 28 28 F F F 29 580-599 14 14 14 28 28 28 F F F Max gen: 16, Max flush: 30 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 17 17 17 31 31 31 F F F 6 120-139 14 14 14 27 27 27 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 17 17 17 31 31 31 F F F 10 200-219 17 17 17 31 31 31 F F F 11 220-239 17 17 17 31 31 31 F F F 12 240-259 13 13 13 25 25 25 F F F 13 260-279 17 17 17 31 31 31 F F F 14 280-299 16 16 16 30 30 30 F F F 15 300-319 13 13 13 25 25 25 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 16 16 16 30 30 30 F F F 19 380-399 14 14 14 27 27 27 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 17 17 17 31 31 31 F F F 23 460-479 17 17 17 31 31 31 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 10 10 10 18 18 18 F F F 28 560-579 14 14 14 28 28 28 F F F 29 580-599 14 14 14 28 28 28 F F F Max gen: 17, Max flush: 31 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.697728282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.705717388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.705996033Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.70602099Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706029759Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706041147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Crucible 23354c24-28fe-4a08-904a-d8cc1f3a6be2 has session id: e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706392483Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Upstairs opts: Upstairs UUID: 23354c24-28fe-4a08-904a-d8cc1f3a6be2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706422309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Crucible stats registered with UUID: 23354c24-28fe-4a08-904a-d8cc1f3a6be2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706433578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:12.706442756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.70656939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706600686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706611795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706748898Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706765985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"23354c24-28fe-4a08-904a-d8cc1f3a6be2 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706784033Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706792902Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"0","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706802511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"1","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.7068118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"2","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706821658Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706858834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706881101Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51892)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706891909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:35259)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706902448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45696)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.706921406Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707542887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"0","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707576403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"1","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707587671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"2","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707873495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[0]R flush_numbers[0..12]: [25, 30, 19, 19, 31, 31, 27, 27, 30, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707899932Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[0]R generation[0..12]: [13, 16, 11, 11, 17, 17, 14, 14, 16, 17, 17, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.70791082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707921929Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[1]R flush_numbers[0..12]: [25, 30, 19, 19, 31, 31, 27, 27, 30, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707932338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[1]R generation[0..12]: [13, 16, 11, 11, 17, 17, 14, 14, 16, 17, 17, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707942446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707953205Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[2]R flush_numbers[0..12]: [25, 30, 19, 19, 31, 31, 27, 27, 30, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707963634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[2]R generation[0..12]: [13, 16, 11, 11, 17, 17, 14, 14, 16, 17, 17, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707973872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.707984741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.7079941Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708004278Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708015297Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708024746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"":"downstairs","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708034635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708043573Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"0","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708053502Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"1","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708063241Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"client":"2","":"downstairs","session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.70807325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708082568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"23354c24-28fe-4a08-904a-d8cc1f3a6be2 is now active with session: e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708092397Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708101066Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"session_id":"e85cbf0a-4b65-4d82-81e4-f9398e9ac4eb"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708110085Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Activated sub_volume 23354c24-28fe-4a08-904a-d8cc1f3a6be2","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708118024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708126063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.708420006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"task":"crutest"} 01/30 Write block 209 len 20480 data: 3 3 3 3 3 02/30 Flush 03/30 Write block 129 len 16384 data: 3 3 4 4 04/30 Read block 214 len 24576 05/30 Read block 66 len 32768 06/30 Write block 207 len 36864 data: 2 3 4 4 4 4 4 3 3 07/30 Write block 262 len 20480 data: 3 3 3 3 3 08/30 Write block 296 len 4096 data: 3 09/30 Read block 207 len 28672 10/30 Read block 242 len 4096 11/30 Write block 177 len 4096 data: 4 12/30 Write block 141 len 40960 data: 5 5 4 4 3 3 3 3 3 3 13/30 Read block 554 len 4096 14/30 Write block 180 len 4096 data: 3 15/30 Read block 323 len 12288 16/30 Write block 557 len 28672 data: 2 3 3 3 2 2 3 17/30 Read block 89 len 32768 18/30 Read block 350 len 8192 19/30 Read block 376 len 20480 20/30 Read block 45 len 40960 21/30 Write block 446 len 36864 data: 4 4 3 3 3 3 4 4 4 22/30 Write block 303 len 24576 data: 3 3 3 3 3 2 23/30 Read block 140 len 36864 24/30 Flush 25/30 Write block 375 len 28672 data: 4 3 3 3 2 2 2 26/30 Read block 278 len 24576 27/30 Write block 146 len 36864 data: 4 4 4 4 4 2 2 2 2 28/30 Read block 235 len 4096 29/30 Write block 387 len 32768 data: 2 2 4 4 4 5 5 5 30/30 Write block 258 len 28672 data: 4 5 4 3 4 4 4 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 25 29 Done 26 26 3 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:12.717182934Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:12.717337375Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 17 17 17 31 31 31 F F F 6 120-139 18 18 14 33 33 27 F F T <--- 7 140-159 18 18 14 33 33 27 T T F <--- 8 160-179 18 18 16 33 33 30 F F F <--- 9 180-199 18 18 17 33 33 31 F F F <--- 10 200-219 18 18 18 33 33 32 F F F <--- 11 220-239 17 17 17 31 31 31 F F F 12 240-259 13 13 13 25 25 25 T T F <--- 13 260-279 18 18 17 33 33 31 T T F <--- 14 280-299 18 18 16 33 33 30 F F F <--- 15 300-319 18 18 13 33 33 25 F F F <--- 16 320-339 7 7 7 12 12 12 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 16 16 16 30 30 30 T T F <--- 19 380-399 14 14 14 27 27 27 T T F <--- 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 18 18 17 33 33 31 F F F <--- 23 460-479 17 17 17 31 31 31 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 18 18 10 33 33 18 F F F <--- 28 560-579 18 18 14 33 33 28 F F F <--- 29 580-599 14 14 14 28 28 28 F F F Max gen: 18, Max flush: 33 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 17 17 17 31 31 31 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 34 34 34 F F F 8 160-179 18 18 18 33 33 33 F F F 9 180-199 18 18 18 33 33 33 F F F 10 200-219 18 18 18 33 33 33 F F F 11 220-239 17 17 17 31 31 31 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 18 18 18 33 33 33 F F F 16 320-339 7 7 7 12 12 12 F F F 17 340-359 15 15 15 29 29 29 F F F 18 360-379 19 19 19 34 34 34 F F F 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 18 18 18 33 33 33 F F F 23 460-479 17 17 17 31 31 31 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 18 18 18 33 33 33 F F F 28 560-579 18 18 18 33 33 33 F F F 29 580-599 14 14 14 28 28 28 F F F Max gen: 19, Max flush: 34 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.967650443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.975654498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.97595062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.975986185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.975998514Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976010402Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Crucible 71e6fe93-8f5c-4cc3-ae43-201ee3d2c039 has session id: 49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976338351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Upstairs opts: Upstairs UUID: 71e6fe93-8f5c-4cc3-ae43-201ee3d2c039, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976376316Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Crucible stats registered with UUID: 71e6fe93-8f5c-4cc3-ae43-201ee3d2c039","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976387005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:26.976396183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976540445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976556983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976567032Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976724942Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.97674139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"71e6fe93-8f5c-4cc3-ae43-201ee3d2c039 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976758877Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976767816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"0","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976779625Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"1","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976789214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"2","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976798652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976808531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.97681854Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33528)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976858105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41653)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976869024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45305)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.976879272Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977459779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"0","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977553867Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"1","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977573714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"2","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977829582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[0]R flush_numbers[0..12]: [25, 30, 19, 19, 31, 31, 33, 34, 33, 33, 33, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977848409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[0]R generation[0..12]: [13, 16, 11, 11, 17, 17, 18, 19, 18, 18, 18, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977858808Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977869737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[1]R flush_numbers[0..12]: [25, 30, 19, 19, 31, 31, 33, 34, 33, 33, 33, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977882265Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[1]R generation[0..12]: [13, 16, 11, 11, 17, 17, 18, 19, 18, 18, 18, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977892514Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977903342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[2]R flush_numbers[0..12]: [25, 30, 19, 19, 31, 31, 33, 34, 33, 33, 33, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977913661Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[2]R generation[0..12]: [13, 16, 11, 11, 17, 17, 18, 19, 18, 18, 18, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.97792381Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977934648Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977944087Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977953746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977963065Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977972354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"":"downstairs","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977982132Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.977990751Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"0","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.97800032Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"1","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978009879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"client":"2","":"downstairs","session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978019958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978030756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"71e6fe93-8f5c-4cc3-ae43-201ee3d2c039 is now active with session: 49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978040605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978049274Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"session_id":"49fe33ae-3deb-4cdf-8bd6-79dd4844ae2f"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978058213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Activated sub_volume 71e6fe93-8f5c-4cc3-ae43-201ee3d2c039","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978066042Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978074051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.978367573Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"task":"crutest"} 01/30 Write block 163 len 12288 data: 5 5 3 02/30 Write block 555 len 36864 data: 2 2 3 4 4 4 3 3 4 03/30 Write block 573 len 8192 data: 4 4 04/30 Read block 496 len 8192 05/30 Write block 187 len 4096 data: 2 06/30 Read block 135 len 40960 07/30 Read block 411 len 8192 08/30 Flush 09/30 Write block 541 len 20480 data: 4 3 3 3 3 10/30 Read block 277 len 40960 11/30 Read block 178 len 12288 12/30 Write block 152 len 40960 data: 3 3 3 2 4 4 4 3 3 4 13/30 Write block 208 len 4096 data: 4 14/30 Write block 26 len 40960 data: 2 2 2 3 3 3 3 3 3 3 15/30 Read block 481 len 36864 16/30 Read block 457 len 40960 17/30 Read block 347 len 40960 18/30 Read block 199 len 8192 19/30 Read block 87 len 24576 20/30 Read block 88 len 4096 21/30 Read block 415 len 8192 22/30 Read block 438 len 36864 23/30 Read block 44 len 24576 24/30 Write block 144 len 20480 data: 5 4 5 5 5 25/30 Write block 462 len 16384 data: 3 4 4 4 26/30 Read block 456 len 28672 27/30 Write block 341 len 4096 data: 2 28/30 Write block 308 len 16384 data: 3 2 3 3 29/30 Write block 319 len 28672 data: 2 2 2 2 2 2 2 30/30 Read block 535 len 16384 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 26 27 Done 30 29 4 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:26.985977028Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:26.986156726Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1222,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 16 16 16 30 30 30 T T F <--- 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 17 17 17 31 31 31 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 19 19 19 34 34 34 T T F <--- 8 160-179 20 20 18 35 35 33 T T T <--- 9 180-199 20 20 18 35 35 33 F F F <--- 10 200-219 18 18 18 33 33 33 T T F <--- 11 220-239 17 17 17 31 31 31 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 18 18 18 33 33 33 T T F <--- 16 320-339 7 7 7 12 12 12 T T F <--- 17 340-359 15 15 15 29 29 29 T T F <--- 18 360-379 19 19 19 34 34 34 F F F 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 18 18 18 33 33 33 F F F 23 460-479 17 17 17 31 31 31 T T F <--- 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 20 20 18 35 35 33 T T T <--- 28 560-579 20 20 18 35 35 33 F F T <--- 29 580-599 14 14 14 28 28 28 F F F Max gen: 20, Max flush: 35 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 F F F 1 020-039 21 21 21 36 36 36 F F F 2 040-059 11 11 11 19 19 19 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 17 17 17 31 31 31 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 21 21 21 36 36 36 F F F 8 160-179 21 21 21 36 36 36 F F F 9 180-199 20 20 20 35 35 35 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 17 17 17 31 31 31 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 21 21 21 36 36 36 F F F 16 320-339 21 21 21 36 36 36 F F F 17 340-359 21 21 21 36 36 36 F F F 18 360-379 19 19 19 34 34 34 F F F 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 18 18 18 33 33 33 F F F 23 460-479 21 21 21 36 36 36 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 12 12 12 21 21 21 F F F 27 540-559 21 21 21 36 36 36 F F F 28 560-579 20 20 20 35 35 35 F F F 29 580-599 14 14 14 28 28 28 F F F Max gen: 21, Max flush: 36 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.23555795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.243448619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.243729323Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.24375256Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.243761549Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.243772968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Crucible 3b2e5ad3-4f59-440e-a014-eca30e7d63a9 has session id: 864ff894-91fc-4c2a-820d-9c39768ddc4e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244203583Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Upstairs opts: Upstairs UUID: 3b2e5ad3-4f59-440e-a014-eca30e7d63a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.24423229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Crucible stats registered with UUID: 3b2e5ad3-4f59-440e-a014-eca30e7d63a9","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244248557Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:41.244257976Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244372022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244396439Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244407197Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244552349Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244568967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"3b2e5ad3-4f59-440e-a014-eca30e7d63a9 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244582235Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244590934Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"0","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244600433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"1","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244609952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"2","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.24461931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244629259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244638998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50911)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244709559Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58222)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244734466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42183)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.244747904Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245440916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"0","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245514297Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"1","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245528695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"2","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245821788Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[0]R flush_numbers[0..12]: [25, 36, 19, 19, 31, 31, 33, 36, 36, 35, 36, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245840736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[0]R generation[0..12]: [13, 21, 11, 11, 17, 17, 18, 21, 21, 20, 21, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245851304Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245862293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[1]R flush_numbers[0..12]: [25, 36, 19, 19, 31, 31, 33, 36, 36, 35, 36, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245872522Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[1]R generation[0..12]: [13, 21, 11, 11, 17, 17, 18, 21, 21, 20, 21, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.24588269Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245893649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[2]R flush_numbers[0..12]: [25, 36, 19, 19, 31, 31, 33, 36, 36, 35, 36, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245903998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[2]R generation[0..12]: [13, 21, 11, 11, 17, 17, 18, 21, 21, 20, 21, 17]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245914156Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245924925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245937213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245946822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245956101Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.24596529Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"":"downstairs","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245974949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245983637Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"0","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.245993176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"1","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246002565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"client":"2","":"downstairs","session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246012354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246021383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"3b2e5ad3-4f59-440e-a014-eca30e7d63a9 is now active with session: 864ff894-91fc-4c2a-820d-9c39768ddc4e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246031121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.24603982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"session_id":"864ff894-91fc-4c2a-820d-9c39768ddc4e"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246048769Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Activated sub_volume 3b2e5ad3-4f59-440e-a014-eca30e7d63a9","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246056698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246064667Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.246372868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"task":"crutest"} 01/30 Write block 361 len 40960 data: 3 2 3 4 3 3 3 4 4 4 02/30 Write block 167 len 40960 data: 4 5 5 5 5 4 4 4 5 4 03/30 Write block 574 len 32768 data: 5 3 3 3 3 3 3 3 04/30 Read block 535 len 20480 05/30 Flush 06/30 Write block 474 len 28672 data: 3 4 3 3 2 2 2 07/30 Write block 220 len 36864 data: 3 3 3 3 3 2 2 2 3 08/30 Read block 226 len 40960 09/30 Write block 450 len 20480 data: 4 4 5 5 5 10/30 Read block 505 len 36864 11/30 Read block 526 len 20480 12/30 Read block 506 len 8192 13/30 Write block 114 len 8192 data: 3 3 14/30 Write block 539 len 20480 data: 6 5 5 4 4 15/30 Read block 81 len 40960 16/30 Write block 232 len 20480 data: 3 3 2 2 3 17/30 Flush 18/30 Write block 575 len 12288 data: 4 4 4 19/30 Read block 539 len 12288 20/30 Write block 146 len 32768 data: 6 6 6 5 5 3 4 4 21/30 Read block 274 len 4096 22/30 Flush 23/30 Write block 148 len 12288 data: 7 6 6 24/30 Read block 476 len 16384 25/30 Read block 252 len 28672 26/30 Write block 13 len 28672 data: 3 3 3 3 3 3 2 27/30 Read block 173 len 40960 28/30 Read block 155 len 32768 29/30 Read block 107 len 8192 30/30 Write block 49 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 26 28 Done 29 29 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:41.255317624Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:41.255485382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1231,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 25 25 25 T T F <--- 1 020-039 21 21 21 36 36 36 F F F 2 040-059 11 11 11 19 19 19 T T F <--- 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 22 22 17 38 38 31 F F F <--- 6 120-139 18 18 18 33 33 33 F F F 7 140-159 22 22 21 39 39 36 T T F <--- 8 160-179 22 22 21 37 37 36 F F T <--- 9 180-199 20 20 20 35 35 35 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 22 22 17 38 38 31 F F F <--- 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 21 21 21 36 36 36 F F F 16 320-339 21 21 21 36 36 36 F F F 17 340-359 21 21 21 36 36 36 F F F 18 360-379 22 22 19 37 37 34 F F T <--- 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 22 22 18 38 38 33 F F F <--- 23 460-479 22 22 21 38 38 36 F F F <--- 24 480-499 22 22 9 38 38 15 F F F <--- 25 500-519 13 13 13 25 25 25 F F F 26 520-539 22 22 12 38 38 21 F F F <--- 27 540-559 22 22 21 38 38 36 F F F <--- 28 560-579 22 22 20 39 39 35 F F T <--- 29 580-599 22 22 14 37 37 28 F F T <--- Max gen: 22, Max flush: 39 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 40 40 40 F F F 1 020-039 21 21 21 36 36 36 F F F 2 040-059 23 23 23 40 40 40 F F F 3 060-079 11 11 11 19 19 19 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 22 22 22 38 38 38 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 23 23 23 40 40 40 F F F 8 160-179 22 22 22 37 37 37 F F F 9 180-199 20 20 20 35 35 35 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 22 22 22 38 38 38 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 21 21 21 36 36 36 F F F 16 320-339 21 21 21 36 36 36 F F F 17 340-359 21 21 21 36 36 36 F F F 18 360-379 22 22 22 37 37 37 F F F 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 22 22 22 38 38 38 F F F 23 460-479 22 22 22 38 38 38 F F F 24 480-499 22 22 22 38 38 38 F F F 25 500-519 13 13 13 25 25 25 F F F 26 520-539 22 22 22 38 38 38 F F F 27 540-559 22 22 22 38 38 38 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 22 22 22 37 37 37 F F F Max gen: 23, Max flush: 40 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.505775284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.513761781Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514076951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514116256Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514124985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514136243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Crucible 7a1bd7bc-b9c3-4b93-b8f8-46b181ce5030 has session id: 634cfb25-4d37-4748-8bf7-a8b6778d72ba","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514457403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Upstairs opts: Upstairs UUID: 7a1bd7bc-b9c3-4b93-b8f8-46b181ce5030, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514524384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Crucible stats registered with UUID: 7a1bd7bc-b9c3-4b93-b8f8-46b181ce5030","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514536013Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:55.514545342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514677135Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514693693Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514703971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514855672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514878519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"7a1bd7bc-b9c3-4b93-b8f8-46b181ce5030 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514887368Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514898447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"0","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514908605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"1","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514918364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"2","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514927893Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514937922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.51494778Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:58412)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514984246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:62006)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.514995084Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:33435)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.51503019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.5156643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"0","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.515703445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"1","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.515806532Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"2","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516094265Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[0]R flush_numbers[0..12]: [40, 36, 40, 19, 31, 38, 33, 40, 37, 35, 36, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516115062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[0]R generation[0..12]: [23, 21, 23, 11, 17, 22, 18, 23, 22, 20, 21, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516125751Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516139019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[1]R flush_numbers[0..12]: [40, 36, 40, 19, 31, 38, 33, 40, 37, 35, 36, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516149468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[1]R generation[0..12]: [23, 21, 23, 11, 17, 22, 18, 23, 22, 20, 21, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516159787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516170615Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[2]R flush_numbers[0..12]: [40, 36, 40, 19, 31, 38, 33, 40, 37, 35, 36, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516180844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[2]R generation[0..12]: [23, 21, 23, 11, 17, 22, 18, 23, 22, 20, 21, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516190943Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516201761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.51621107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516220929Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516230318Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516239607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"":"downstairs","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516249185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516257764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"0","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516267213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"1","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516278442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"client":"2","":"downstairs","session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516288061Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516297159Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"7a1bd7bc-b9c3-4b93-b8f8-46b181ce5030 is now active with session: 634cfb25-4d37-4748-8bf7-a8b6778d72ba","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516306918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516315557Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"session_id":"634cfb25-4d37-4748-8bf7-a8b6778d72ba"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516324516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Activated sub_volume 7a1bd7bc-b9c3-4b93-b8f8-46b181ce5030","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516332445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516340614Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.516618189Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"task":"crutest"} 01/30 Write block 463 len 4096 data: 5 02/30 Read block 521 len 20480 03/30 Write block 508 len 20480 data: 2 2 2 2 3 04/30 Write block 19 len 28672 data: 3 2 2 2 2 2 2 05/30 Read block 235 len 16384 06/30 Read block 178 len 28672 07/30 Write block 62 len 20480 data: 2 3 3 3 3 08/30 Read block 218 len 8192 09/30 Read block 35 len 32768 10/30 Read block 448 len 16384 11/30 Read block 113 len 32768 12/30 Write block 213 len 20480 data: 5 4 4 4 3 13/30 Write block 164 len 28672 data: 6 4 3 5 6 6 6 14/30 Flush 15/30 Write block 151 len 40960 data: 4 5 5 4 3 5 5 5 4 4 16/30 Read block 588 len 16384 17/30 Read block 540 len 36864 18/30 Read block 523 len 36864 19/30 Read block 460 len 12288 20/30 Write block 553 len 12288 data: 2 2 3 21/30 Read block 66 len 20480 22/30 Read block 404 len 36864 23/30 Read block 379 len 24576 24/30 Read block 415 len 32768 25/30 Read block 88 len 12288 26/30 Read block 161 len 20480 27/30 Read block 124 len 16384 28/30 Write block 101 len 4096 data: 4 29/30 Write block 186 len 16384 data: 4 3 2 2 30/30 Write block 295 len 16384 data: 3 4 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 30 36 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:47:55.525052379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:47:55.52519854Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1240,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 23 41 41 40 F F F <--- 1 020-039 24 24 21 41 41 36 F F F <--- 2 040-059 23 23 23 40 40 40 F F F 3 060-079 24 24 11 41 41 19 F F F <--- 4 080-099 17 17 17 31 31 31 F F F 5 100-119 22 22 22 38 38 38 T T F <--- 6 120-139 18 18 18 33 33 33 F F F 7 140-159 23 23 23 40 40 40 T T F <--- 8 160-179 24 24 22 41 41 37 T T F <--- 9 180-199 20 20 20 35 35 35 T T F <--- 10 200-219 24 24 21 41 41 36 F F F <--- 11 220-239 22 22 22 38 38 38 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 18 18 18 33 33 33 T T F <--- 15 300-319 21 21 21 36 36 36 F F F 16 320-339 21 21 21 36 36 36 F F F 17 340-359 21 21 21 36 36 36 F F F 18 360-379 22 22 22 37 37 37 F F F 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 22 22 22 38 38 38 F F F 23 460-479 24 24 22 41 41 38 F F F <--- 24 480-499 22 22 22 38 38 38 F F F 25 500-519 24 24 13 41 41 25 F F F <--- 26 520-539 22 22 22 38 38 38 F F F 27 540-559 22 22 22 38 38 38 T T F <--- 28 560-579 22 22 22 39 39 39 F F F 29 580-599 22 22 22 37 37 37 F F F Max gen: 24, Max flush: 41 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 23 23 23 40 40 40 F F F 3 060-079 24 24 24 41 41 41 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 25 25 25 42 42 42 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 25 25 25 42 42 42 F F F 8 160-179 25 25 25 42 42 42 F F F 9 180-199 25 25 25 42 42 42 F F F 10 200-219 24 24 24 41 41 41 F F F 11 220-239 22 22 22 38 38 38 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 25 25 25 42 42 42 F F F 15 300-319 21 21 21 36 36 36 F F F 16 320-339 21 21 21 36 36 36 F F F 17 340-359 21 21 21 36 36 36 F F F 18 360-379 22 22 22 37 37 37 F F F 19 380-399 19 19 19 34 34 34 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 22 22 22 38 38 38 F F F 23 460-479 24 24 24 41 41 41 F F F 24 480-499 22 22 22 38 38 38 F F F 25 500-519 24 24 24 41 41 41 F F F 26 520-539 22 22 22 38 38 38 F F F 27 540-559 25 25 25 42 42 42 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 22 22 22 37 37 37 F F F Max gen: 25, Max flush: 42 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.773381539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.781383074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.781671318Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.781696464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.781705083Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.781716402Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Crucible 8d7f4f08-b680-46bc-a29b-1ae8856342a7 has session id: 33d380cf-6291-41e7-ba8e-1b946cb80f56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782063518Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Upstairs opts: Upstairs UUID: 8d7f4f08-b680-46bc-a29b-1ae8856342a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782102043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Crucible stats registered with UUID: 8d7f4f08-b680-46bc-a29b-1ae8856342a7","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782115151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:09.78212434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782270042Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782301098Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782311806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.78244028Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782455828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"8d7f4f08-b680-46bc-a29b-1ae8856342a7 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782471736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782480455Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"0","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782490104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"1","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782499823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"2","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782509251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782536688Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782568734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"ds_connection connected from Ok(127.0.0.1:57584)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782579992Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"ds_connection connected from Ok(127.0.0.1:33617)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782592801Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"ds_connection connected from Ok(127.0.0.1:34510)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.782603369Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783285813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"1","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.7833061Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"0","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783317379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"2","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783646047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[0]R flush_numbers[0..12]: [41, 41, 40, 41, 31, 42, 33, 42, 42, 42, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783671214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[0]R generation[0..12]: [24, 24, 23, 24, 17, 25, 18, 25, 25, 25, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783682153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783692981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[1]R flush_numbers[0..12]: [41, 41, 40, 41, 31, 42, 33, 42, 42, 42, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.78370329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[1]R generation[0..12]: [24, 24, 23, 24, 17, 25, 18, 25, 25, 25, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783713729Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783724417Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[2]R flush_numbers[0..12]: [41, 41, 40, 41, 31, 42, 33, 42, 42, 42, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783734816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[2]R generation[0..12]: [24, 24, 23, 24, 17, 25, 18, 25, 25, 25, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783745115Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783758173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783767432Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783777011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783786229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783795518Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"":"downstairs","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783805387Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783814156Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"0","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783824015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"1","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783833803Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"client":"2","":"downstairs","session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783843692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783852711Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"8d7f4f08-b680-46bc-a29b-1ae8856342a7 is now active with session: 33d380cf-6291-41e7-ba8e-1b946cb80f56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.78386256Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783871089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"session_id":"33d380cf-6291-41e7-ba8e-1b946cb80f56"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783880078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Activated sub_volume 8d7f4f08-b680-46bc-a29b-1ae8856342a7","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783888117Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.783896206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.784199677Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"task":"crutest"} 01/30 Write block 376 len 36864 data: 4 4 4 3 3 3 2 2 2 02/30 Read block 354 len 16384 03/30 Write block 425 len 24576 data: 4 4 4 3 3 3 04/30 Read block 335 len 8192 05/30 Read block 136 len 20480 06/30 Read block 512 len 12288 07/30 Read block 111 len 12288 08/30 Read block 257 len 12288 09/30 Flush 10/30 Write block 335 len 16384 data: 2 2 2 2 11/30 Read block 426 len 8192 12/30 Write block 293 len 16384 data: 3 3 4 5 13/30 Read block 129 len 16384 14/30 Write block 198 len 40960 data: 3 3 3 3 3 3 3 2 2 3 15/30 Write block 592 len 32768 data: 3 3 3 3 3 2 2 2 16/30 Read block 52 len 32768 17/30 Read block 156 len 20480 18/30 Read block 80 len 4096 19/30 Read block 144 len 32768 20/30 Write block 473 len 20480 data: 3 4 5 4 4 21/30 Read block 229 len 8192 22/30 Write block 332 len 16384 data: 2 2 2 3 23/30 Read block 376 len 4096 24/30 Read block 67 len 40960 25/30 Write block 334 len 40960 data: 3 4 3 3 3 2 2 3 2 3 26/30 Write block 71 len 4096 data: 5 27/30 Write block 146 len 36864 data: 7 7 8 7 7 5 6 6 5 28/30 Read block 348 len 28672 29/30 Write block 141 len 8192 data: 6 6 30/30 Read block 260 len 36864 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 24 24 Done 30 30 6 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:09.793267827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:09.793442685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1249,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 23 23 23 40 40 40 F F F 3 060-079 24 24 24 41 41 41 T T F <--- 4 080-099 17 17 17 31 31 31 F F F 5 100-119 25 25 25 42 42 42 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 25 25 25 42 42 42 T T F <--- 8 160-179 25 25 25 42 42 42 F F F 9 180-199 25 25 25 42 42 42 T T F <--- 10 200-219 24 24 24 41 41 41 T T F <--- 11 220-239 22 22 22 38 38 38 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 25 25 25 42 42 42 T T F <--- 15 300-319 21 21 21 36 36 36 F F F 16 320-339 21 21 21 36 36 36 T T F <--- 17 340-359 21 21 21 36 36 36 T T F <--- 18 360-379 26 26 22 43 43 37 F F T <--- 19 380-399 26 26 19 43 43 34 F F T <--- 20 400-419 9 9 9 15 15 15 F F F 21 420-439 26 26 14 43 43 27 F F T <--- 22 440-459 22 22 22 38 38 38 F F F 23 460-479 24 24 24 41 41 41 T T F <--- 24 480-499 22 22 22 38 38 38 F F F 25 500-519 24 24 24 41 41 41 F F F 26 520-539 22 22 22 38 38 38 F F F 27 540-559 25 25 25 42 42 42 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 22 22 22 37 37 37 T T F <--- Max gen: 26, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 23 23 23 40 40 40 F F F 3 060-079 27 27 27 44 44 44 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 25 25 25 42 42 42 F F F 6 120-139 18 18 18 33 33 33 F F F 7 140-159 27 27 27 44 44 44 F F F 8 160-179 25 25 25 42 42 42 F F F 9 180-199 27 27 27 44 44 44 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 22 22 22 38 38 38 F F F 12 240-259 19 19 19 34 34 34 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 21 21 21 36 36 36 F F F 16 320-339 27 27 27 44 44 44 F F F 17 340-359 27 27 27 44 44 44 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 26 26 26 43 43 43 F F F 22 440-459 22 22 22 38 38 38 F F F 23 460-479 27 27 27 44 44 44 F F F 24 480-499 22 22 22 38 38 38 F F F 25 500-519 24 24 24 41 41 41 F F F 26 520-539 22 22 22 38 38 38 F F F 27 540-559 25 25 25 42 42 42 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 27, Max flush: 44 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.047323141Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055281421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055558676Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055585743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055594382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.05560596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Crucible b9a8f988-6971-4c72-b743-ec471d7b2339 has session id: 9ecd849a-0c6e-41a0-b0e5-cb9566783faa","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055946157Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Upstairs opts: Upstairs UUID: b9a8f988-6971-4c72-b743-ec471d7b2339, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055978853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Crucible stats registered with UUID: b9a8f988-6971-4c72-b743-ec471d7b2339","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.055993051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:24.05600247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056150341Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"0","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056166799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"1","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056176958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"2","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056321919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056339497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"b9a8f988-6971-4c72-b743-ec471d7b2339 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056357475Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056366254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"0","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056375782Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"1","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056385251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"2","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.05639473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"2","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056404649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"0","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056414598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"1","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:54032)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056482099Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"2","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:35481)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056591345Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"1","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:39298)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.056615392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"io task","client":"0","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057163633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"0","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057203637Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"1","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057214796Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"2","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057524377Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[0]R flush_numbers[0..12]: [41, 41, 40, 44, 31, 42, 33, 44, 42, 44, 44, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057544564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[0]R generation[0..12]: [24, 24, 23, 27, 17, 25, 18, 27, 25, 27, 27, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057555223Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057566071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[1]R flush_numbers[0..12]: [41, 41, 40, 44, 31, 42, 33, 44, 42, 44, 44, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.05757648Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[1]R generation[0..12]: [24, 24, 23, 27, 17, 25, 18, 27, 25, 27, 27, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057587019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057597907Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[2]R flush_numbers[0..12]: [41, 41, 40, 44, 31, 42, 33, 44, 42, 44, 44, 38]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057608126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[2]R generation[0..12]: [24, 24, 23, 27, 17, 25, 18, 27, 25, 27, 27, 22]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057618495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057629363Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057638662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057648301Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.05765753Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057666799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"":"downstairs","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057676377Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057685046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"0","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057696215Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"1","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057705714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"client":"2","":"downstairs","session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057715433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057724421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"b9a8f988-6971-4c72-b743-ec471d7b2339 is now active with session: 9ecd849a-0c6e-41a0-b0e5-cb9566783faa","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.05773416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057742769Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"session_id":"9ecd849a-0c6e-41a0-b0e5-cb9566783faa"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057751718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Activated sub_volume b9a8f988-6971-4c72-b743-ec471d7b2339","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057759847Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.057767706Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.058074057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"task":"crutest"} 01/30 Write block 100 len 20480 data: 3 5 3 3 3 02/30 Read block 100 len 24576 03/30 Write block 251 len 32768 data: 4 5 5 4 4 4 4 5 04/30 Write block 307 len 20480 data: 4 4 3 4 4 05/30 Read block 519 len 8192 06/30 Read block 278 len 16384 07/30 Write block 163 len 4096 data: 6 08/30 Flush 09/30 Write block 538 len 40960 data: 4 7 6 6 5 5 4 4 3 2 10/30 Read block 135 len 36864 11/30 Write block 51 len 36864 data: 3 3 2 2 2 2 2 2 2 12/30 Write block 547 len 24576 data: 3 2 2 3 3 2 13/30 Flush 14/30 Write block 502 len 4096 data: 3 15/30 Read block 549 len 28672 16/30 Write block 67 len 36864 data: 3 3 3 3 6 5 4 5 6 17/30 Read block 105 len 8192 18/30 Read block 75 len 40960 19/30 Read block 355 len 40960 20/30 Flush 21/30 Write block 57 len 24576 data: 3 3 3 2 2 3 22/30 Read block 178 len 24576 23/30 Read block 571 len 16384 24/30 Flush 25/30 Write block 172 len 4096 data: 5 26/30 Read block 266 len 24576 27/30 Write block 412 len 20480 data: 2 2 2 2 2 28/30 Write block 118 len 28672 data: 2 2 2 2 3 3 2 29/30 Write block 217 len 24576 data: 4 3 3 4 4 4 30/30 Read block 55 len 28672 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:24.067006684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:24.067176843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1258,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 28 23 28 48 40 48 F F F <--- 3 060-079 28 27 28 48 44 48 F F F <--- 4 080-099 17 17 17 31 31 31 F F F 5 100-119 28 25 28 45 42 45 T F T <--- 6 120-139 18 18 18 33 33 33 T F T <--- 7 140-159 27 27 27 44 44 44 F F F 8 160-179 28 25 28 45 42 45 T F T <--- 9 180-199 27 27 27 44 44 44 F F F 10 200-219 27 27 27 44 44 44 T F T <--- 11 220-239 22 22 22 38 38 38 T F T <--- 12 240-259 28 19 28 45 34 45 F F F <--- 13 260-279 19 19 19 34 34 34 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 28 21 28 45 36 45 F F F <--- 16 320-339 27 27 27 44 44 44 F F F 17 340-359 27 27 27 44 44 44 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 9 9 9 15 15 15 T F T <--- 21 420-439 26 26 26 43 43 43 F F F 22 440-459 22 22 22 38 38 38 F F F 23 460-479 27 27 27 44 44 44 F F F 24 480-499 22 22 22 38 38 38 F F F 25 500-519 28 24 28 47 41 47 F F F <--- 26 520-539 28 22 28 46 38 46 F F F <--- 27 540-559 28 25 28 46 42 46 F F F <--- 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 28, Max flush: 48 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 17 17 17 31 31 31 F F F 5 100-119 29 29 29 49 49 49 F F F 6 120-139 29 29 29 49 49 49 F F F 7 140-159 27 27 27 44 44 44 F F F 8 160-179 29 29 29 49 49 49 F F F 9 180-199 27 27 27 44 44 44 F F F 10 200-219 29 29 29 49 49 49 F F F 11 220-239 29 29 29 49 49 49 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 28 28 28 45 45 45 F F F 16 320-339 27 27 27 44 44 44 F F F 17 340-359 27 27 27 44 44 44 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 29 29 29 49 49 49 F F F 21 420-439 26 26 26 43 43 43 F F F 22 440-459 22 22 22 38 38 38 F F F 23 460-479 27 27 27 44 44 44 F F F 24 480-499 22 22 22 38 38 38 F F F 25 500-519 28 28 28 47 47 47 F F F 26 520-539 28 28 28 46 46 46 F F F 27 540-559 28 28 28 46 46 46 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 29, Max flush: 49 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.318697398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.326655209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.326945232Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.326970019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.326978668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.326989986Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Crucible fc944396-7aee-4335-8446-7a4ca01da150 has session id: e91ea773-41ba-459e-8bc4-d99170670dbd","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327381317Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Upstairs opts: Upstairs UUID: fc944396-7aee-4335-8446-7a4ca01da150, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327414552Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Crucible stats registered with UUID: fc944396-7aee-4335-8446-7a4ca01da150","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327425751Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:38.32743495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.32758718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327616757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327627975Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327754049Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327774437Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"fc944396-7aee-4335-8446-7a4ca01da150 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327790945Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327799774Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"0","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327809312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"1","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327818811Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"2","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.32782831Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327838239Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327848177Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41999)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327895071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36249)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327916149Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50001)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.327927037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328481587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"0","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328517013Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"1","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328544609Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"2","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328840722Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[0]R flush_numbers[0..12]: [41, 41, 48, 48, 31, 49, 49, 44, 49, 44, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328865938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[0]R generation[0..12]: [24, 24, 28, 28, 17, 29, 29, 27, 29, 27, 29, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328876877Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328887866Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[1]R flush_numbers[0..12]: [41, 41, 48, 48, 31, 49, 49, 44, 49, 44, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328898124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[1]R generation[0..12]: [24, 24, 28, 28, 17, 29, 29, 27, 29, 27, 29, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328908323Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328919232Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[2]R flush_numbers[0..12]: [41, 41, 48, 48, 31, 49, 49, 44, 49, 44, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.32893169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[2]R generation[0..12]: [24, 24, 28, 28, 17, 29, 29, 27, 29, 27, 29, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328942209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328952987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328962336Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328971885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328981074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328990373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"":"downstairs","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.328999991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.32900873Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"0","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329018299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"1","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329027838Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"client":"2","":"downstairs","session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329037517Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329046565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"fc944396-7aee-4335-8446-7a4ca01da150 is now active with session: e91ea773-41ba-459e-8bc4-d99170670dbd","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329056354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329064933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"session_id":"e91ea773-41ba-459e-8bc4-d99170670dbd"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329074032Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Activated sub_volume fc944396-7aee-4335-8446-7a4ca01da150","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329083231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.32909116Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.329398581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"task":"crutest"} 01/30 Write block 102 len 40960 data: 4 4 4 4 5 6 7 6 5 4 02/30 Write block 515 len 12288 data: 4 4 4 03/30 Read block 314 len 24576 04/30 Read block 265 len 20480 05/30 Read block 393 len 16384 06/30 Read block 131 len 40960 07/30 Flush 08/30 Write block 492 len 8192 data: 3 3 09/30 Read block 144 len 32768 10/30 Read block 355 len 32768 11/30 Read block 530 len 16384 12/30 Read block 102 len 8192 13/30 Write block 440 len 40960 data: 3 2 2 2 2 3 5 5 4 4 14/30 Flush 15/30 Write block 418 len 36864 data: 2 2 2 4 3 4 4 5 5 16/30 Read block 264 len 12288 17/30 Write block 99 len 28672 data: 3 4 6 5 5 5 5 18/30 Read block 106 len 12288 19/30 Read block 500 len 36864 20/30 Read block 561 len 40960 21/30 Write block 164 len 12288 data: 7 5 4 22/30 Write block 41 len 16384 data: 3 3 3 3 23/30 Write block 49 len 40960 data: 4 3 4 4 3 3 3 3 4 4 24/30 Read block 286 len 8192 25/30 Write block 180 len 16384 data: 4 3 3 6 26/30 Flush 27/30 Write block 503 len 24576 data: 3 3 3 2 2 3 28/30 Read block 365 len 8192 29/30 Write block 131 len 20480 data: 5 5 4 4 5 30/30 Read block 377 len 12288 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 23 23 Done 23 23 0 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:38.338930872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:38.339110599Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1267,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 30 30 28 52 52 48 F F F <--- 3 060-079 28 28 28 48 48 48 F F F 4 080-099 30 30 17 52 52 31 F F F <--- 5 100-119 30 30 30 52 52 50 F F F <--- 6 120-139 29 29 29 49 49 49 T T F <--- 7 140-159 27 27 27 44 44 44 F F F 8 160-179 30 30 29 52 52 49 F F F <--- 9 180-199 30 30 27 52 52 44 F F F <--- 10 200-219 29 29 29 49 49 49 F F F 11 220-239 29 29 29 49 49 49 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 28 28 28 45 45 45 F F F 16 320-339 27 27 27 44 44 44 F F F 17 340-359 27 27 27 44 44 44 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 30 30 29 52 52 49 F F F <--- 21 420-439 30 30 26 52 52 43 F F F <--- 22 440-459 30 30 22 51 51 38 F F F <--- 23 460-479 27 27 27 44 44 44 F F F 24 480-499 30 30 22 51 51 38 F F F <--- 25 500-519 30 30 30 50 50 50 T T F <--- 26 520-539 28 28 28 46 46 46 F F F 27 540-559 28 28 28 46 46 46 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 30, Max flush: 52 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 30 30 30 52 52 52 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 31 31 31 53 53 53 F F F 7 140-159 27 27 27 44 44 44 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 29 29 29 49 49 49 F F F 11 220-239 29 29 29 49 49 49 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 27 27 27 44 44 44 F F F 15 300-319 28 28 28 45 45 45 F F F 16 320-339 27 27 27 44 44 44 F F F 17 340-359 27 27 27 44 44 44 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 30 30 30 51 51 51 F F F 23 460-479 27 27 27 44 44 44 F F F 24 480-499 30 30 30 51 51 51 F F F 25 500-519 31 31 31 53 53 53 F F F 26 520-539 28 28 28 46 46 46 F F F 27 540-559 28 28 28 46 46 46 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 31, Max flush: 53 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.590988249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.598998333Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599321942Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599346859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599355408Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599366976Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Crucible 24cecff0-657d-4f72-8635-b281f1a1caa3 has session id: 7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.59973566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Upstairs opts: Upstairs UUID: 24cecff0-657d-4f72-8635-b281f1a1caa3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599764696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Crucible stats registered with UUID: 24cecff0-657d-4f72-8635-b281f1a1caa3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599776204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:52.599785623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599928005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.599958851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.59997012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600110362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60012651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"24cecff0-657d-4f72-8635-b281f1a1caa3 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600160496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600169825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"0","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600179493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"1","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600188992Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"2","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600198401Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60020833Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600218178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"ds_connection connected from Ok(127.0.0.1:33045)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60028341Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"ds_connection connected from Ok(127.0.0.1:53510)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60036403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"ds_connection connected from Ok(127.0.0.1:41783)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.600386937Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601039594Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"0","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601062641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"2","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601109215Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"1","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601397899Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[0]R flush_numbers[0..12]: [41, 41, 52, 48, 52, 52, 53, 44, 52, 52, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601416426Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[0]R generation[0..12]: [24, 24, 30, 28, 30, 30, 31, 27, 30, 30, 29, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601426915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601437934Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[1]R flush_numbers[0..12]: [41, 41, 52, 48, 52, 52, 53, 44, 52, 52, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601448222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[1]R generation[0..12]: [24, 24, 30, 28, 30, 30, 31, 27, 30, 30, 29, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601458491Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60146939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[2]R flush_numbers[0..12]: [41, 41, 52, 48, 52, 52, 53, 44, 52, 52, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601479828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[2]R generation[0..12]: [24, 24, 30, 28, 30, 30, 31, 27, 30, 30, 29, 29]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601490057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601500886Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601510165Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601519903Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601529252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601538561Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"":"downstairs","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60154855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601557469Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"0","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601568567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"1","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601578086Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"client":"2","":"downstairs","session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601587885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601597104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"24cecff0-657d-4f72-8635-b281f1a1caa3 is now active with session: 7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601607142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601615811Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"session_id":"7f02adb1-47bf-4ecd-9d0b-bc02d6f1b453"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.60162487Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Activated sub_volume 24cecff0-657d-4f72-8635-b281f1a1caa3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601632989Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601641038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.601947289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"task":"crutest"} 01/30 Write block 481 len 28672 data: 2 2 2 2 3 3 3 02/30 Read block 590 len 24576 03/30 Read block 84 len 28672 04/30 Read block 136 len 32768 05/30 Read block 85 len 36864 06/30 Write block 318 len 4096 data: 2 07/30 Write block 299 len 36864 data: 3 3 3 3 4 4 4 4 5 08/30 Write block 548 len 32768 data: 3 3 4 4 3 3 3 4 09/30 Write block 536 len 28672 data: 4 4 5 8 7 7 6 10/30 Flush 11/30 Write block 9 len 16384 data: 2 2 3 3 12/30 Write block 120 len 16384 data: 3 3 4 4 13/30 Write block 221 len 40960 data: 5 5 4 4 3 3 3 4 3 3 14/30 Write block 387 len 28672 data: 3 3 5 5 5 6 6 15/30 Read block 551 len 40960 16/30 Write block 531 len 16384 data: 4 5 5 4 17/30 Read block 8 len 12288 18/30 Write block 338 len 24576 data: 4 3 3 4 3 4 19/30 Read block 357 len 40960 20/30 Read block 326 len 12288 21/30 Read block 597 len 4096 22/30 Read block 101 len 36864 23/30 Write block 289 len 32768 data: 3 3 2 2 4 4 5 6 24/30 Write block 143 len 12288 data: 5 6 5 25/30 Write block 468 len 20480 data: 4 4 4 4 4 26/30 Read block 486 len 20480 27/30 Read block 180 len 8192 28/30 Read block 263 len 4096 29/30 Read block 472 len 40960 30/30 Write block 220 len 32768 data: 5 6 6 5 5 4 4 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:48:52.610489266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:48:52.610666523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1276,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 41 41 41 T F T <--- 1 020-039 24 24 24 41 41 41 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 30 30 30 52 52 52 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 31 31 31 53 53 53 T F T <--- 7 140-159 27 27 27 44 44 44 T F T <--- 8 160-179 30 30 30 52 52 52 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 29 29 29 49 49 49 F F F 11 220-239 29 29 29 49 49 49 T F T <--- 12 240-259 28 28 28 45 45 45 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 32 27 32 54 44 54 T F T <--- 15 300-319 32 28 32 54 45 54 F F F <--- 16 320-339 27 27 27 44 44 44 T F T <--- 17 340-359 27 27 27 44 44 44 T F T <--- 18 360-379 26 26 26 43 43 43 F F F 19 380-399 26 26 26 43 43 43 T F T <--- 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 30 30 30 51 51 51 F F F 23 460-479 27 27 27 44 44 44 T F T <--- 24 480-499 32 30 32 54 51 54 F T F <--- 25 500-519 31 31 31 53 53 53 F F F 26 520-539 32 28 32 54 46 54 T F T <--- 27 540-559 32 28 32 54 46 54 F F F <--- 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 32, Max flush: 54 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 24 24 24 41 41 41 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 30 30 30 52 52 52 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 33 33 33 55 55 55 F F F 7 140-159 33 33 33 55 55 55 F F F 8 160-179 30 30 30 52 52 52 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 29 29 29 49 49 49 F F F 11 220-239 33 33 33 55 55 55 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 19 19 19 34 34 34 F F F 14 280-299 33 33 33 55 55 55 F F F 15 300-319 32 32 32 54 54 54 F F F 16 320-339 33 33 33 55 55 55 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 33 33 33 55 55 55 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 30 30 52 52 52 F F F 22 440-459 30 30 30 51 51 51 F F F 23 460-479 33 33 33 55 55 55 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 31 31 31 53 53 53 F F F 26 520-539 33 33 33 55 55 55 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 33, Max flush: 55 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.86722167Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:06.87431152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.874366543Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881146934Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881398682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881433857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881446296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881458084Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Crucible 3ae53cb6-dcbc-41f5-a317-b678ad0f1e11 has session id: 60b7396a-0d38-4e32-a58a-e5f5f497753b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881758056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Upstairs opts: Upstairs UUID: 3ae53cb6-dcbc-41f5-a317-b678ad0f1e11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881821068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Crucible stats registered with UUID: 3ae53cb6-dcbc-41f5-a317-b678ad0f1e11","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881832887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:06.881842405Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881976838Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.881993006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882002965Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882150346Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882166744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"3ae53cb6-dcbc-41f5-a317-b678ad0f1e11 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882184692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882193671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"0","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.88220336Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"1","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882213058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"2","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882224627Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882234716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882268861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53027)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882285949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33403)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882296398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53552)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882321185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.882884213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"1","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.88290868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"2","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883032534Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"0","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883327207Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[0]R flush_numbers[0..12]: [55, 41, 52, 48, 52, 52, 55, 55, 52, 52, 49, 55]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883362193Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[0]R generation[0..12]: [33, 24, 30, 28, 30, 30, 33, 33, 30, 30, 29, 33]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883373011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.88338397Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[1]R flush_numbers[0..12]: [55, 41, 52, 48, 52, 52, 55, 55, 52, 52, 49, 55]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883394308Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[1]R generation[0..12]: [33, 24, 30, 28, 30, 30, 33, 33, 30, 30, 29, 33]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883407357Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883418325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[2]R flush_numbers[0..12]: [55, 41, 52, 48, 52, 52, 55, 55, 52, 52, 49, 55]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883428634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[2]R generation[0..12]: [33, 24, 30, 28, 30, 30, 33, 33, 30, 30, 29, 33]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883438863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883449511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.88345903Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883468739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883477998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883487267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"":"downstairs","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883496855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883505424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"0","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883515103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"1","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883524722Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"client":"2","":"downstairs","session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883534591Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.88354368Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"3ae53cb6-dcbc-41f5-a317-b678ad0f1e11 is now active with session: 60b7396a-0d38-4e32-a58a-e5f5f497753b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883553578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883563907Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"session_id":"60b7396a-0d38-4e32-a58a-e5f5f497753b"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883572756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Activated sub_volume 3ae53cb6-dcbc-41f5-a317-b678ad0f1e11","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883581035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883589244Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.883835373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} 01/30 Write block 169 len 32768 data: 7 7 6 6 5 5 6 5 02/30 Write block 441 len 4096 data: 3 03/30 Read block 379 len 36864 04/30 Flush 05/30 Write block 481 len 20480 data: 3 3 3 3 4 06/30 Write block 367 len 4096 data: 4 07/30 Write block 33 len 32768 data: 4 4 4 3 3 3 2 3 08/30 Flush 09/30 Write block 459 len 8192 data: 4 3 10/30 Read block 221 len 40960 11/30 Write block 107 len 28672 data: 7 8 7 6 5 4 4 12/30 Write block 428 len 8192 data: 4 4 13/30 Read block 416 len 12288 14/30 Write block 82 len 32768 data: 2 3 4 5 5 5 6 6 15/30 Read block 498 len 16384 16/30 Write block 533 len 16384 data: 6 5 4 5 17/30 Write block 131 len 16384 data: 6 6 5 5 18/30 Read block 198 len 28672 19/30 Read block 260 len 16384 20/30 Write block 287 len 4096 data: 4 21/30 Read block 556 len 8192 22/30 Write block 203 len 24576 data: 4 4 3 3 4 5 23/30 Write block 279 len 8192 data: 5 5 24/30 Write block 508 len 24576 data: 4 3 3 3 4 3 25/30 Read block 392 len 32768 26/30 Flush 27/30 Write block 530 len 24576 data: 4 5 6 7 6 5 28/30 Read block 207 len 24576 29/30 Read block 555 len 24576 30/30 Write block 42 len 12288 data: 4 4 4 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:06.892148368Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:06.892331595Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 24 34 34 41 57 57 F F F <--- 2 040-059 30 34 34 52 57 57 F T T <--- 3 060-079 28 28 28 48 48 48 F F F 4 080-099 30 34 34 52 58 58 F F F <--- 5 100-119 30 34 34 52 58 58 F F F <--- 6 120-139 33 34 34 55 58 58 F F F <--- 7 140-159 33 33 33 55 55 55 F F F 8 160-179 30 34 34 52 56 56 F F F <--- 9 180-199 30 30 30 52 52 52 F F F 10 200-219 29 34 34 49 58 58 F F F <--- 11 220-239 33 33 33 55 55 55 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 19 34 34 34 58 58 F F F <--- 14 280-299 33 34 34 55 58 58 F F F <--- 15 300-319 32 32 32 54 54 54 F F F 16 320-339 33 33 33 55 55 55 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 26 34 34 43 57 57 F F F <--- 19 380-399 33 33 33 55 55 55 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 30 34 34 52 58 58 F F F <--- 22 440-459 30 34 34 51 58 58 F F F <--- 23 460-479 33 34 34 55 58 58 F F F <--- 24 480-499 32 34 34 54 57 57 F F F <--- 25 500-519 31 34 34 53 58 58 F F F <--- 26 520-539 33 34 34 55 58 58 F T T <--- 27 540-559 32 32 32 54 54 54 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 34, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 35 35 35 59 59 59 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 33 33 33 55 55 55 F F F 8 160-179 34 34 34 56 56 56 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 34 34 34 58 58 58 F F F 11 220-239 33 33 33 55 55 55 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 32 32 32 54 54 54 F F F 16 320-339 33 33 33 55 55 55 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 33 33 33 55 55 55 F F F 20 400-419 30 30 30 52 52 52 F F F 21 420-439 34 34 34 58 58 58 F F F 22 440-459 34 34 34 58 58 58 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 35 35 35 59 59 59 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 35, Max flush: 59 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.152104554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.160301154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.160600986Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.160625593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.160634152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.16064584Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Crucible 6c862313-1369-4691-b5d1-925b500c1906 has session id: 0853939e-2d81-4bc3-855d-fbf4d23d3d58","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161060488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Upstairs opts: Upstairs UUID: 6c862313-1369-4691-b5d1-925b500c1906, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161088894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Crucible stats registered with UUID: 6c862313-1369-4691-b5d1-925b500c1906","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161100073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:21.161110891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161254213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161285229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161296358Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161413153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161430351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"6c862313-1369-4691-b5d1-925b500c1906 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161447219Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161455948Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"0","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161465856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"1","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161475575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"2","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161485094Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161495043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161505201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:33340)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161580682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:37490)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.16159852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:39841)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.161609148Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162292642Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"1","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162322668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"2","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162334056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"0","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162655946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[0]R flush_numbers[0..12]: [55, 57, 59, 48, 58, 58, 58, 55, 56, 52, 58, 55]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162681582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[0]R generation[0..12]: [33, 34, 35, 28, 34, 34, 34, 33, 34, 30, 34, 33]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162692311Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.16270324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[1]R flush_numbers[0..12]: [55, 57, 59, 48, 58, 58, 58, 55, 56, 52, 58, 55]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162713828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[1]R generation[0..12]: [33, 34, 35, 28, 34, 34, 34, 33, 34, 30, 34, 33]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162724097Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162734905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[2]R flush_numbers[0..12]: [55, 57, 59, 48, 58, 58, 58, 55, 56, 52, 58, 55]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162745204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[2]R generation[0..12]: [33, 34, 35, 28, 34, 34, 34, 33, 34, 30, 34, 33]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162755383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162766192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.16277545Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162786739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162796138Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162805527Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"":"downstairs","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162815075Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162823784Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"0","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162833313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"1","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162842912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"client":"2","":"downstairs","session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162852531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162861539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"6c862313-1369-4691-b5d1-925b500c1906 is now active with session: 0853939e-2d81-4bc3-855d-fbf4d23d3d58","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162871468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162880107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"session_id":"0853939e-2d81-4bc3-855d-fbf4d23d3d58"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162889126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Activated sub_volume 6c862313-1369-4691-b5d1-925b500c1906","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162901744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.162909933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.163160682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"task":"crutest"} 01/30 Write block 229 len 12288 data: 4 4 3 02/30 Write block 412 len 40960 data: 3 3 3 3 3 2 3 3 3 5 03/30 Write block 72 len 36864 data: 6 5 6 7 5 5 5 5 3 04/30 Write block 303 len 28672 data: 5 5 5 5 6 5 4 05/30 Read block 406 len 24576 06/30 Write block 264 len 8192 data: 5 4 07/30 Write block 275 len 12288 data: 3 4 5 08/30 Read block 19 len 24576 09/30 Read block 256 len 20480 10/30 Read block 479 len 8192 11/30 Read block 528 len 24576 12/30 Write block 536 len 28672 data: 6 5 6 9 8 8 7 13/30 Read block 516 len 40960 14/30 Flush 15/30 Write block 188 len 36864 data: 3 3 3 3 3 3 3 3 3 16/30 Read block 354 len 8192 17/30 Write block 523 len 32768 data: 2 2 4 4 4 4 4 5 18/30 Read block 390 len 8192 19/30 Write block 431 len 16384 data: 3 3 3 3 20/30 Read block 311 len 8192 21/30 Read block 45 len 8192 22/30 Flush 23/30 Write block 327 len 40960 data: 3 3 3 3 2 3 3 4 5 4 24/30 Write block 392 len 4096 data: 7 25/30 Read block 333 len 8192 26/30 Read block 257 len 36864 27/30 Read block 13 len 12288 28/30 Read block 8 len 20480 29/30 Read block 458 len 16384 30/30 Read block 403 len 16384 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 7 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:21.171817004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:21.171994061Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1294,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 35 35 35 59 59 59 F F F 3 060-079 36 36 28 60 60 48 F F T <--- 4 080-099 36 36 34 60 60 58 F F T <--- 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 33 33 33 55 55 55 F F F 8 160-179 34 34 34 56 56 56 F F F 9 180-199 36 36 30 61 61 52 F F F <--- 10 200-219 34 34 34 58 58 58 F F F 11 220-239 36 36 33 60 60 55 F F T <--- 12 240-259 28 28 28 45 45 45 F F F 13 260-279 36 36 34 60 60 58 F F F <--- 14 280-299 34 34 34 58 58 58 F F F 15 300-319 36 36 32 60 60 54 F F F <--- 16 320-339 33 33 33 55 55 55 T T F <--- 17 340-359 33 33 33 55 55 55 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 33 33 33 55 55 55 T T F <--- 20 400-419 36 36 30 60 60 52 F F T <--- 21 420-439 36 36 34 61 61 58 F F T <--- 22 440-459 34 34 34 58 58 58 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 36 36 35 61 61 59 F F F <--- 27 540-559 36 36 32 60 60 54 F F F <--- 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 36, Max flush: 61 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 35 35 35 59 59 59 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 36 36 36 60 60 60 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 33 33 33 55 55 55 F F F 8 160-179 34 34 34 56 56 56 F F F 9 180-199 36 36 36 61 61 61 F F F 10 200-219 34 34 34 58 58 58 F F F 11 220-239 36 36 36 60 60 60 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 36 36 36 61 61 61 F F F 22 440-459 34 34 34 58 58 58 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 36 36 36 61 61 61 F F F 27 540-559 36 36 36 60 60 60 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.425543889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:35.432524564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.432572998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439319952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439577379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439601696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439609925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439621314Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Crucible 4ef5719d-c2d2-4ce5-a8db-501b22b2bdbd has session id: 35a0d624-693d-48be-9920-7f1e880e8033","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439970399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Upstairs opts: Upstairs UUID: 4ef5719d-c2d2-4ce5-a8db-501b22b2bdbd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439987347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Crucible stats registered with UUID: 4ef5719d-c2d2-4ce5-a8db-501b22b2bdbd","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.439997396Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:35.440010594Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440155286Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440171894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440181973Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440331054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440343952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"4ef5719d-c2d2-4ce5-a8db-501b22b2bdbd active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.44035767Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440366739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"0","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440376588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"1","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440386267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"2","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440395785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440405764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440415733Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"ds_connection connected from Ok(127.0.0.1:63767)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440497313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"ds_connection connected from Ok(127.0.0.1:43421)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440522399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"ds_connection connected from Ok(127.0.0.1:56942)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.440610268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441202623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"1","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441215212Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"0","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441258296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"2","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441550239Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[0]R flush_numbers[0..12]: [55, 57, 59, 60, 60, 58, 58, 55, 56, 61, 58, 60]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441580385Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[0]R generation[0..12]: [33, 34, 35, 36, 36, 34, 34, 33, 34, 36, 34, 36]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441593204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441604222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[1]R flush_numbers[0..12]: [55, 57, 59, 60, 60, 58, 58, 55, 56, 61, 58, 60]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441614571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[1]R generation[0..12]: [33, 34, 35, 36, 36, 34, 34, 33, 34, 36, 34, 36]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.44162471Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441635718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[2]R flush_numbers[0..12]: [55, 57, 59, 60, 60, 58, 58, 55, 56, 61, 58, 60]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441646037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[2]R generation[0..12]: [33, 34, 35, 36, 36, 34, 34, 33, 34, 36, 34, 36]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441656376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441667364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441676763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441686422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441695721Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441705129Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"":"downstairs","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441714728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441723297Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"0","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441732906Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"1","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441743744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"client":"2","":"downstairs","session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441753643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441762542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"4ef5719d-c2d2-4ce5-a8db-501b22b2bdbd is now active with session: 35a0d624-693d-48be-9920-7f1e880e8033","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441772261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.44178104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"session_id":"35a0d624-693d-48be-9920-7f1e880e8033"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441790219Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Activated sub_volume 4ef5719d-c2d2-4ce5-a8db-501b22b2bdbd","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441798488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.441806696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.442063724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} 01/30 Write block 214 len 4096 data: 5 02/30 Write block 451 len 4096 data: 5 03/30 Flush 04/30 Write block 443 len 12288 data: 3 3 4 05/30 Read block 443 len 28672 06/30 Write block 168 len 40960 data: 7 8 8 7 7 6 6 7 6 5 07/30 Write block 125 len 20480 data: 2 2 3 3 4 08/30 Write block 199 len 40960 data: 4 4 4 4 5 5 4 4 5 6 09/30 Write block 248 len 4096 data: 3 10/30 Write block 117 len 8192 data: 2 3 11/30 Write block 139 len 16384 data: 7 5 7 7 12/30 Read block 308 len 4096 13/30 Write block 453 len 16384 data: 6 6 3 3 14/30 Read block 480 len 36864 15/30 Read block 397 len 4096 16/30 Write block 281 len 36864 data: 5 5 5 3 4 4 5 4 4 17/30 Read block 62 len 32768 18/30 Write block 534 len 28672 data: 7 6 7 6 7 10 9 19/30 Flush 20/30 Write block 422 len 24576 data: 4 5 5 6 6 5 21/30 Read block 421 len 8192 22/30 Read block 31 len 28672 23/30 Read block 372 len 32768 24/30 Read block 400 len 20480 25/30 Write block 407 len 4096 data: 3 26/30 Read block 95 len 12288 27/30 Write block 584 len 40960 data: 3 3 3 3 3 2 2 3 4 4 28/30 Read block 263 len 24576 29/30 Read block 329 len 24576 30/30 Read block 337 len 8192 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 3 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:35.450600081Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:35.450753382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1303,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 35 35 35 59 59 59 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 36 36 36 60 60 60 F F F 5 100-119 34 38 38 58 64 64 F F F <--- 6 120-139 34 38 38 58 64 64 F F F <--- 7 140-159 33 38 38 55 64 64 F F F <--- 8 160-179 34 38 38 56 64 64 F F F <--- 9 180-199 36 38 38 61 64 64 F F F <--- 10 200-219 38 38 38 63 64 64 F F F <--- 11 220-239 36 36 36 60 60 60 F F F 12 240-259 28 38 38 45 64 64 F F F <--- 13 260-279 36 36 36 60 60 60 F F F 14 280-299 34 38 38 58 64 64 F F F <--- 15 300-319 36 36 36 60 60 60 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 36 36 36 60 60 60 F T T <--- 21 420-439 36 36 36 61 61 61 F T T <--- 22 440-459 38 38 38 63 64 64 F F F <--- 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 36 38 38 61 64 64 F F F <--- 27 540-559 36 38 38 60 64 64 F F F <--- 28 560-579 22 22 22 39 39 39 F F F 29 580-599 27 27 27 44 44 44 F T T <--- Max gen: 38, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 35 35 35 59 59 59 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 36 36 36 60 60 60 F F F 5 100-119 38 38 38 64 64 64 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 64 64 64 F F F 9 180-199 38 38 38 64 64 64 F F F 10 200-219 38 38 38 64 64 64 F F F 11 220-239 36 36 36 60 60 60 F F F 12 240-259 38 38 38 64 64 64 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 38 38 38 64 64 64 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 39 39 39 65 65 65 F F F 21 420-439 39 39 39 65 65 65 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 38 38 38 64 64 64 F F F 27 540-559 38 38 38 64 64 64 F F F 28 560-579 22 22 22 39 39 39 F F F 29 580-599 39 39 39 65 65 65 F F F Max gen: 39, Max flush: 65 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.700691578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.70864002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.708916905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.708944111Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.70895271Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.708964159Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Crucible 6a5c8db5-8acf-4037-9a19-3a266dfe2a43 has session id: 9aaec0b0-7c83-495e-aa08-923cc75e830b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709302496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Upstairs opts: Upstairs UUID: 6a5c8db5-8acf-4037-9a19-3a266dfe2a43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709333962Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Crucible stats registered with UUID: 6a5c8db5-8acf-4037-9a19-3a266dfe2a43","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.70934765Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:49.709357159Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709470324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709501571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709512459Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709644702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.70966064Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"6a5c8db5-8acf-4037-9a19-3a266dfe2a43 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709684077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709693446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"0","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709703095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"1","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709712644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"2","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709722173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709731931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.70974183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33679)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.70981879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62185)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709831809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57026)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.709849656Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710498794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"0","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710519301Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"1","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710534639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"2","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710914091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[0]R flush_numbers[0..12]: [55, 57, 59, 60, 60, 64, 64, 64, 64, 64, 64, 60]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710931619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[0]R generation[0..12]: [33, 34, 35, 36, 36, 38, 38, 38, 38, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710941978Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710952806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[1]R flush_numbers[0..12]: [55, 57, 59, 60, 60, 64, 64, 64, 64, 64, 64, 60]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710963205Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[1]R generation[0..12]: [33, 34, 35, 36, 36, 38, 38, 38, 38, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710973454Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710984282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[2]R flush_numbers[0..12]: [55, 57, 59, 60, 60, 64, 64, 64, 64, 64, 64, 60]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.710994391Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[2]R generation[0..12]: [33, 34, 35, 36, 36, 38, 38, 38, 38, 38, 38, 36]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.71100476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711018578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711028087Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711037636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711046845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711056083Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"":"downstairs","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711065902Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711074511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"0","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.71108409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"1","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711093499Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"client":"2","":"downstairs","session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711103307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711112436Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"6a5c8db5-8acf-4037-9a19-3a266dfe2a43 is now active with session: 9aaec0b0-7c83-495e-aa08-923cc75e830b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711122305Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711131004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"session_id":"9aaec0b0-7c83-495e-aa08-923cc75e830b"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711139943Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Activated sub_volume 6a5c8db5-8acf-4037-9a19-3a266dfe2a43","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711147822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711155961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.711462332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"task":"crutest"} 01/30 Flush 02/30 Write block 555 len 20480 data: 5 3 4 5 5 03/30 Read block 515 len 36864 04/30 Write block 252 len 20480 data: 6 6 5 5 5 05/30 Write block 248 len 8192 data: 4 3 06/30 Write block 48 len 32768 data: 3 5 4 5 5 4 4 4 07/30 Read block 122 len 36864 08/30 Read block 223 len 16384 09/30 Write block 53 len 16384 data: 5 5 5 4 10/30 Flush 11/30 Flush 12/30 Write block 10 len 12288 data: 3 4 4 13/30 Write block 503 len 12288 data: 4 4 4 14/30 Write block 575 len 36864 data: 5 5 5 4 4 4 4 3 3 15/30 Write block 212 len 40960 data: 5 6 6 5 5 5 4 4 6 7 16/30 Write block 329 len 8192 data: 4 4 17/30 Write block 39 len 16384 data: 3 4 4 5 18/30 Write block 474 len 28672 data: 5 6 5 5 3 3 3 19/30 Read block 442 len 12288 20/30 Read block 186 len 40960 21/30 Write block 375 len 24576 data: 5 5 5 5 4 4 22/30 Write block 564 len 12288 data: 2 2 2 23/30 Write block 544 len 4096 data: 5 24/30 Read block 17 len 12288 25/30 Write block 414 len 16384 data: 4 4 4 3 26/30 Read block 215 len 24576 27/30 Write block 325 len 16384 data: 3 2 4 4 28/30 Read block 449 len 36864 29/30 Write block 538 len 32768 data: 8 11 10 9 8 6 6 5 30/30 Read block 573 len 24576 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:49:49.7192844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:49:49.719450199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1312,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 55 55 55 T T F <--- 1 020-039 34 34 34 57 57 57 T T F <--- 2 040-059 40 40 35 67 67 59 T T F <--- 3 060-079 36 36 36 60 60 60 F F F 4 080-099 36 36 36 60 60 60 F F F 5 100-119 38 38 38 64 64 64 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 64 64 64 F F F 9 180-199 38 38 38 64 64 64 F F F 10 200-219 38 38 38 64 64 64 T T F <--- 11 220-239 36 36 36 60 60 60 T T F <--- 12 240-259 40 40 38 67 67 64 F F F <--- 13 260-279 36 36 36 60 60 60 F F F 14 280-299 38 38 38 64 64 64 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 37 37 37 62 62 62 T T F <--- 17 340-359 33 33 33 55 55 55 F F F 18 360-379 34 34 34 57 57 57 T T F <--- 19 380-399 37 37 37 62 62 62 T T F <--- 20 400-419 39 39 39 65 65 65 T T F <--- 21 420-439 39 39 39 65 65 65 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 34 34 34 58 58 58 T T F <--- 24 480-499 34 34 34 57 57 57 T T F <--- 25 500-519 34 34 34 58 58 58 T T F <--- 26 520-539 38 38 38 64 64 64 T T F <--- 27 540-559 40 40 38 67 67 64 T T T <--- 28 560-579 22 22 22 39 39 39 T T F <--- 29 580-599 39 39 39 65 65 65 T T F <--- Max gen: 40, Max flush: 67 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 41 41 41 68 68 68 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 36 36 36 60 60 60 F F F 5 100-119 38 38 38 64 64 64 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 64 64 64 F F F 9 180-199 38 38 38 64 64 64 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 41 41 41 68 68 68 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 38 38 38 64 64 64 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 41 41 41 68 68 68 F F F 19 380-399 41 41 41 68 68 68 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 39 39 39 65 65 65 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 41 41 41 68 68 68 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 41 41 41 68 68 68 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.988586629Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.996571267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.996852861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.996877398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.996885797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.996897395Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Crucible a4950458-27c7-4a22-8ac8-5221de7df1db has session id: 86bce9f0-40c2-42b8-89ab-35abf17d7f56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997322602Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Upstairs opts: Upstairs UUID: a4950458-27c7-4a22-8ac8-5221de7df1db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997425598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Crucible stats registered with UUID: a4950458-27c7-4a22-8ac8-5221de7df1db","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997439597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:03.997448646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.99757015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997586738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997596737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997758906Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997771685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"a4950458-27c7-4a22-8ac8-5221de7df1db active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997787343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997796221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"0","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.99780579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"1","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997815319Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"2","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997824898Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997834787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997844615Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"ds_connection connected from Ok(127.0.0.1:62798)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997904938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"ds_connection connected from Ok(127.0.0.1:64803)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997930194Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"ds_connection connected from Ok(127.0.0.1:33199)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.997941343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.998616967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"0","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.998636015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"1","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.99867718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"2","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.998971802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[0]R flush_numbers[0..12]: [68, 68, 68, 60, 60, 64, 64, 64, 64, 64, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.99899257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[0]R generation[0..12]: [41, 41, 41, 36, 36, 38, 38, 38, 38, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999003178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999014377Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[1]R flush_numbers[0..12]: [68, 68, 68, 60, 60, 64, 64, 64, 64, 64, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999024756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[1]R generation[0..12]: [41, 41, 41, 36, 36, 38, 38, 38, 38, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999034954Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999045743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[2]R flush_numbers[0..12]: [68, 68, 68, 60, 60, 64, 64, 64, 64, 64, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999056002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[2]R generation[0..12]: [41, 41, 41, 36, 36, 38, 38, 38, 38, 38, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.99906633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999077149Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999086468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999096027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999105305Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999114754Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"":"downstairs","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999124423Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999133182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"0","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.99914408Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"1","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999153699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"client":"2","":"downstairs","session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999163618Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999172757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"a4950458-27c7-4a22-8ac8-5221de7df1db is now active with session: 86bce9f0-40c2-42b8-89ab-35abf17d7f56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999182586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999191175Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"session_id":"86bce9f0-40c2-42b8-89ab-35abf17d7f56"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999200113Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Activated sub_volume a4950458-27c7-4a22-8ac8-5221de7df1db","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999208012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999215991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:03.999475169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"task":"crutest"} 01/30 Write block 352 len 20480 data: 4 4 4 4 4 02/30 Read block 358 len 4096 03/30 Read block 111 len 8192 04/30 Write block 280 len 16384 data: 6 6 6 6 05/30 Write block 40 len 32768 data: 5 5 6 5 5 3 4 4 06/30 Read block 255 len 36864 07/30 Read block 494 len 12288 08/30 Read block 162 len 4096 09/30 Read block 101 len 32768 10/30 Read block 421 len 20480 11/30 Read block 460 len 12288 12/30 Write block 286 len 8192 data: 5 6 13/30 Write block 189 len 20480 data: 4 4 4 4 4 14/30 Write block 580 len 28672 data: 5 5 4 4 4 4 4 15/30 Write block 369 len 32768 data: 5 5 4 4 4 4 6 6 16/30 Read block 523 len 36864 17/30 Flush 18/30 Write block 421 len 20480 data: 6 5 6 6 7 19/30 Read block 279 len 12288 20/30 Read block 466 len 24576 21/30 Read block 58 len 40960 22/30 Flush 23/30 Write block 434 len 24576 data: 4 3 3 3 3 3 24/30 Write block 85 len 28672 data: 6 6 6 7 7 6 4 25/30 Write block 182 len 40960 data: 4 7 5 4 5 4 4 5 5 5 26/30 Flush 27/30 Write block 346 len 28672 data: 2 2 2 2 2 2 5 28/30 Write block 292 len 16384 data: 3 5 5 6 29/30 Write block 394 len 8192 data: 6 4 30/30 Read block 27 len 12288 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 18 0 18 Done 30 12 30 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:04.007624185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:04.007795973Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1321,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 42 41 42 69 68 69 F T F <--- 3 060-079 36 36 36 60 60 60 F F F 4 080-099 42 36 42 71 60 71 F F F <--- 5 100-119 38 38 38 64 64 64 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 64 64 64 F F F 9 180-199 42 38 42 71 64 71 F F F <--- 10 200-219 41 41 41 68 68 68 F F F 11 220-239 41 41 41 68 68 68 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 42 38 42 69 64 69 T T T <--- 15 300-319 36 36 36 60 60 60 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 42 33 42 69 55 69 T T T <--- 18 360-379 42 41 42 69 68 69 F F F <--- 19 380-399 41 41 41 68 68 68 T F T <--- 20 400-419 41 41 41 68 68 68 F F F 21 420-439 42 39 42 71 65 71 F F F <--- 22 440-459 38 38 38 64 64 64 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 41 41 41 68 68 68 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 41 41 41 68 68 68 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 42 41 42 69 68 69 F F F <--- Max gen: 42, Max flush: 71 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 42 42 42 71 71 71 F F F 5 100-119 38 38 38 64 64 64 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 64 64 64 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 41 41 41 68 68 68 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 43 43 43 72 72 72 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 42 42 42 71 71 71 F F F 22 440-459 38 38 38 64 64 64 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 41 41 41 68 68 68 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 41 41 41 68 68 68 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 42 42 42 69 69 69 F F F Max gen: 43, Max flush: 72 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.250494337Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:18.25741243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.257461504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264245633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264497432Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264525958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264536777Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264548245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Crucible f982c9be-22d0-4da3-87ef-482da1d5417f has session id: 336174dd-b81d-4d3f-88eb-afbaf4bdb45b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264898221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Upstairs opts: Upstairs UUID: f982c9be-22d0-4da3-87ef-482da1d5417f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264951834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Crucible stats registered with UUID: f982c9be-22d0-4da3-87ef-482da1d5417f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.264965182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:18.264974141Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265104595Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265120702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265131031Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265283772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.26529916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"f982c9be-22d0-4da3-87ef-482da1d5417f active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265309848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265318547Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"0","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265328236Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"1","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265337965Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"2","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265349243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265359242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265369081Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41934)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265415955Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63356)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265427564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44052)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.265437772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266223073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"1","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266266287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"0","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266278146Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"2","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266609584Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[0]R flush_numbers[0..12]: [68, 68, 69, 60, 71, 64, 64, 64, 64, 71, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266632291Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[0]R generation[0..12]: [41, 41, 42, 36, 42, 38, 38, 38, 38, 42, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266642859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266654098Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[1]R flush_numbers[0..12]: [68, 68, 69, 60, 71, 64, 64, 64, 64, 71, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266664497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[1]R generation[0..12]: [41, 41, 42, 36, 42, 38, 38, 38, 38, 42, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266674695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266688364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[2]R flush_numbers[0..12]: [68, 68, 69, 60, 71, 64, 64, 64, 64, 71, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266698802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[2]R generation[0..12]: [41, 41, 42, 36, 42, 38, 38, 38, 38, 42, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266708921Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.26671984Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266729608Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266739177Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266748346Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266757665Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"":"downstairs","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266767424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266776153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"0","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266785921Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"1","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.26679554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"client":"2","":"downstairs","session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266805289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266814348Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"f982c9be-22d0-4da3-87ef-482da1d5417f is now active with session: 336174dd-b81d-4d3f-88eb-afbaf4bdb45b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266824236Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266834435Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"session_id":"336174dd-b81d-4d3f-88eb-afbaf4bdb45b"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266843384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Activated sub_volume f982c9be-22d0-4da3-87ef-482da1d5417f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266851503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.266859572Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.267155164Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} 01/30 Write block 129 len 24576 data: 5 4 7 7 6 6 02/30 Read block 241 len 4096 03/30 Read block 384 len 16384 04/30 Read block 34 len 20480 05/30 Read block 389 len 24576 06/30 Write block 412 len 36864 data: 4 4 5 5 5 4 4 4 4 07/30 Read block 456 len 32768 08/30 Write block 256 len 4096 data: 6 09/30 Flush 10/30 Write block 400 len 8192 data: 4 5 11/30 Read block 257 len 40960 12/30 Read block 411 len 40960 13/30 Read block 188 len 8192 14/30 Write block 108 len 8192 data: 9 8 15/30 Flush 16/30 Write block 408 len 4096 data: 2 17/30 Read block 490 len 4096 18/30 Write block 594 len 24576 data: 4 4 4 3 3 3 19/30 Write block 157 len 32768 data: 6 6 5 5 5 5 7 8 20/30 Write block 167 len 20480 data: 6 8 9 9 8 21/30 Read block 191 len 40960 22/30 Write block 269 len 24576 data: 2 2 2 2 2 2 23/30 Read block 250 len 16384 24/30 Read block 348 len 20480 25/30 Write block 71 len 40960 data: 7 7 6 7 8 6 6 6 6 4 26/30 Write block 172 len 40960 data: 8 7 7 8 7 6 3 3 5 4 27/30 Write block 447 len 16384 data: 6 5 5 5 28/30 Read block 209 len 32768 29/30 Write block 86 len 24576 data: 7 7 8 8 7 5 30/30 Read block 348 len 32768 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 4 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:18.275969597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:18.276137945Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1330,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 36 36 36 60 60 60 F T T <--- 4 080-099 42 42 42 71 71 71 F T T <--- 5 100-119 38 44 44 64 74 74 F F F <--- 6 120-139 38 44 44 64 73 73 F F F <--- 7 140-159 38 38 38 64 64 64 F T T <--- 8 160-179 38 38 38 64 64 64 F T T <--- 9 180-199 42 42 42 71 71 71 F T T <--- 10 200-219 41 41 41 68 68 68 F F F 11 220-239 41 41 41 68 68 68 F F F 12 240-259 40 44 44 67 73 73 F F F <--- 13 260-279 36 36 36 60 60 60 F T T <--- 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 43 43 43 72 72 72 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 41 44 44 68 74 74 F T T <--- 21 420-439 42 44 44 71 73 73 F F F <--- 22 440-459 38 38 38 64 64 64 F T T <--- 23 460-479 41 41 41 68 68 68 F F F 24 480-499 41 41 41 68 68 68 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 41 41 41 68 68 68 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 42 42 42 69 69 69 F T T <--- Max gen: 44, Max flush: 74 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 45 45 45 75 75 75 F F F 4 080-099 45 45 45 75 75 75 F F F 5 100-119 44 44 44 74 74 74 F F F 6 120-139 44 44 44 73 73 73 F F F 7 140-159 45 45 45 75 75 75 F F F 8 160-179 45 45 45 75 75 75 F F F 9 180-199 45 45 45 75 75 75 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 41 41 41 68 68 68 F F F 12 240-259 44 44 44 73 73 73 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 43 43 43 72 72 72 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 45 45 45 75 75 75 F F F 21 420-439 44 44 44 73 73 73 F F F 22 440-459 45 45 45 75 75 75 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 41 41 41 68 68 68 F F F 25 500-519 41 41 41 68 68 68 F F F 26 520-539 41 41 41 68 68 68 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 45 45 45 75 75 75 F F F Max gen: 45, Max flush: 75 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.527595358Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.535486638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.535778061Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.535802818Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.535811257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.535822595Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Crucible 9824f47b-9f69-40fe-b32b-032ecc71712d has session id: 552a8a68-bbe8-411d-a45a-0f1035d2eb00","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536140265Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Upstairs opts: Upstairs UUID: 9824f47b-9f69-40fe-b32b-032ecc71712d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536169241Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Crucible stats registered with UUID: 9824f47b-9f69-40fe-b32b-032ecc71712d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536183269Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:32.536192548Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536322312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.53633951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536349848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536500939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536524926Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"9824f47b-9f69-40fe-b32b-032ecc71712d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536536784Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536545563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"0","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536555152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"1","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536564661Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"2","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.53657424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536584128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536593947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"ds_connection connected from Ok(127.0.0.1:52598)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536674607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"ds_connection connected from Ok(127.0.0.1:64674)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536699454Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"ds_connection connected from Ok(127.0.0.1:64561)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.536800171Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537330624Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"1","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537378968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"0","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537431931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"2","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537719225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[0]R flush_numbers[0..12]: [68, 68, 69, 75, 75, 74, 73, 75, 75, 75, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537739222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[0]R generation[0..12]: [41, 41, 42, 45, 45, 44, 44, 45, 45, 45, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537749581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537760419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[1]R flush_numbers[0..12]: [68, 68, 69, 75, 75, 74, 73, 75, 75, 75, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537770758Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[1]R generation[0..12]: [41, 41, 42, 45, 45, 44, 44, 45, 45, 45, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537780897Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537791635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[2]R flush_numbers[0..12]: [68, 68, 69, 75, 75, 74, 73, 75, 75, 75, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537801964Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[2]R generation[0..12]: [41, 41, 42, 45, 45, 44, 44, 45, 45, 45, 41, 41]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537812293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537823091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.53783246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537842189Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537853757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537863176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"":"downstairs","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537872875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537881554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"0","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537891013Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"1","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537900542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"client":"2","":"downstairs","session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.53791035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537919429Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"9824f47b-9f69-40fe-b32b-032ecc71712d is now active with session: 552a8a68-bbe8-411d-a45a-0f1035d2eb00","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537929268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537937887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"session_id":"552a8a68-bbe8-411d-a45a-0f1035d2eb00"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537946816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Activated sub_volume 9824f47b-9f69-40fe-b32b-032ecc71712d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537954785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.537962874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.538272114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"task":"crutest"} 01/30 Write block 587 len 4096 data: 4 02/30 Read block 354 len 32768 03/30 Read block 311 len 32768 04/30 Write block 338 len 16384 data: 5 4 4 5 05/30 Write block 452 len 32768 data: 6 7 7 4 4 4 4 5 06/30 Write block 125 len 32768 data: 3 3 4 4 6 5 8 8 07/30 Read block 491 len 28672 08/30 Write block 418 len 32768 data: 5 5 5 7 6 7 7 8 09/30 Read block 359 len 8192 10/30 Write block 483 len 16384 data: 4 4 5 4 11/30 Read block 462 len 20480 12/30 Read block 374 len 36864 13/30 Read block 288 len 40960 14/30 Flush 15/30 Write block 530 len 16384 data: 6 6 7 8 16/30 Write block 39 len 40960 data: 4 6 6 7 6 6 4 5 5 4 17/30 Write block 345 len 8192 data: 3 3 18/30 Write block 504 len 20480 data: 5 5 3 3 5 19/30 Read block 508 len 20480 20/30 Write block 149 len 12288 data: 8 8 6 21/30 Read block 403 len 4096 22/30 Read block 302 len 20480 23/30 Read block 23 len 24576 24/30 Flush 25/30 Write block 377 len 28672 data: 6 6 5 5 4 3 3 26/30 Flush 27/30 Write block 81 len 8192 data: 2 3 28/30 Write block 235 len 16384 data: 3 4 3 3 29/30 Read block 436 len 28672 30/30 Read block 38 len 28672 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 4 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:32.546728892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:32.54690327Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1339,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 46 46 41 77 77 68 F F F <--- 2 040-059 46 46 42 77 77 69 F F F <--- 3 060-079 45 45 45 75 75 75 F F F 4 080-099 45 45 45 75 75 75 T T F <--- 5 100-119 44 44 44 74 74 74 F F F 6 120-139 46 46 44 76 76 73 F F F <--- 7 140-159 46 46 45 77 77 75 F F F <--- 8 160-179 45 45 45 75 75 75 F F F 9 180-199 45 45 45 75 75 75 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 41 41 41 68 68 68 T T F <--- 12 240-259 44 44 44 73 73 73 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 46 46 41 76 76 68 F F F <--- 17 340-359 46 46 43 77 77 72 F F F <--- 18 360-379 46 46 42 78 78 69 F F F <--- 19 380-399 46 46 43 78 78 72 F F F <--- 20 400-419 46 46 45 76 76 75 F F F <--- 21 420-439 46 46 44 76 76 73 F F F <--- 22 440-459 46 46 45 76 76 75 F F F <--- 23 460-479 41 41 41 68 68 68 F F F 24 480-499 46 46 41 76 76 68 F F F <--- 25 500-519 46 46 41 77 77 68 F F F <--- 26 520-539 46 46 41 77 77 68 F F F <--- 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 46 46 45 76 76 75 F F T <--- Max gen: 46, Max flush: 78 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 46 46 46 77 77 77 F F F 2 040-059 46 46 46 77 77 77 F F F 3 060-079 45 45 45 75 75 75 F F F 4 080-099 47 47 47 79 79 79 F F F 5 100-119 44 44 44 74 74 74 F F F 6 120-139 46 46 46 76 76 76 F F F 7 140-159 46 46 46 77 77 77 F F F 8 160-179 45 45 45 75 75 75 F F F 9 180-199 45 45 45 75 75 75 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 44 44 44 73 73 73 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 46 46 46 76 76 76 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 46 46 46 78 78 78 F F F 19 380-399 46 46 46 78 78 78 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 46 46 46 76 76 76 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 46 46 46 77 77 77 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 46 46 46 76 76 76 F F F Max gen: 47, Max flush: 79 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.80469306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.812591588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.812868203Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.81289471Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.812905988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.812917567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Crucible b0289303-6dec-40f5-a372-55f0ccc5fc33 has session id: 7761ff11-1a88-4d5b-a4a0-4374bfe5614e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813242676Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Upstairs opts: Upstairs UUID: b0289303-6dec-40f5-a372-55f0ccc5fc33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813258274Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Crucible stats registered with UUID: b0289303-6dec-40f5-a372-55f0ccc5fc33","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813270842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:46.813279761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813416284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813432252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.81344213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.81360049Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813624337Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"b0289303-6dec-40f5-a372-55f0ccc5fc33 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813636946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813645685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"0","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813655163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"1","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813666922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"2","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813676341Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813686159Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813695928Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43207)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813771419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64994)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813792976Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51474)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.813803875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814376842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"2","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814423786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"0","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814441634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"1","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814702531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[0]R flush_numbers[0..12]: [68, 77, 77, 75, 79, 74, 76, 77, 75, 75, 68, 79]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814721798Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[0]R generation[0..12]: [41, 46, 46, 45, 47, 44, 46, 46, 45, 45, 41, 47]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814732187Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814743325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[1]R flush_numbers[0..12]: [68, 77, 77, 75, 79, 74, 76, 77, 75, 75, 68, 79]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814756164Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[1]R generation[0..12]: [41, 46, 46, 45, 47, 44, 46, 46, 45, 45, 41, 47]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814766662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814777451Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[2]R flush_numbers[0..12]: [68, 77, 77, 75, 79, 74, 76, 77, 75, 75, 68, 79]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.81478775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[2]R generation[0..12]: [41, 46, 46, 45, 47, 44, 46, 46, 45, 45, 41, 47]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814798238Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814808897Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814818146Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814827785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814837023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814846272Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"":"downstairs","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814856161Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.8148649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"0","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814874729Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"1","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814884258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"client":"2","":"downstairs","session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814893996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814904685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"b0289303-6dec-40f5-a372-55f0ccc5fc33 is now active with session: 7761ff11-1a88-4d5b-a4a0-4374bfe5614e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814914494Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814923123Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"session_id":"7761ff11-1a88-4d5b-a4a0-4374bfe5614e"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814931961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Activated sub_volume b0289303-6dec-40f5-a372-55f0ccc5fc33","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.81493983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.814947869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.815276188Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"task":"crutest"} 01/30 Write block 534 len 8192 data: 8 7 02/30 Write block 101 len 12288 data: 7 6 6 03/30 Write block 362 len 36864 data: 3 4 5 4 4 5 5 6 6 04/30 Write block 579 len 24576 data: 5 6 6 5 5 5 05/30 Write block 336 len 12288 data: 5 4 6 06/30 Write block 122 len 4096 data: 5 07/30 Write block 247 len 12288 data: 2 5 4 08/30 Read block 150 len 20480 09/30 Write block 586 len 24576 data: 5 5 4 3 3 4 10/30 Read block 450 len 8192 11/30 Read block 403 len 4096 12/30 Read block 395 len 12288 13/30 Read block 227 len 12288 14/30 Write block 141 len 36864 data: 8 8 6 7 6 8 8 9 9 15/30 Write block 567 len 16384 data: 2 2 2 2 16/30 Flush 17/30 Write block 422 len 32768 data: 7 8 8 9 7 6 5 5 18/30 Read block 322 len 16384 19/30 Read block 545 len 32768 20/30 Read block 518 len 20480 21/30 Read block 0 len 32768 22/30 Write block 58 len 4096 data: 5 23/30 Write block 54 len 40960 data: 6 6 5 5 6 4 3 3 4 4 24/30 Write block 393 len 24576 data: 7 7 5 3 3 2 25/30 Write block 192 len 12288 data: 5 5 4 26/30 Write block 2 len 8192 data: 2 2 27/30 Read block 297 len 16384 28/30 Read block 168 len 24576 29/30 Read block 101 len 4096 30/30 Read block 479 len 4096 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:50:46.822684537Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:50:46.822859435Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1348,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 T F T <--- 1 020-039 46 46 46 77 77 77 F F F 2 040-059 46 46 46 77 77 77 T F T <--- 3 060-079 45 45 45 75 75 75 T F T <--- 4 080-099 47 47 47 79 79 79 F F F 5 100-119 48 44 48 80 74 80 F F F <--- 6 120-139 48 46 48 80 76 80 F F F <--- 7 140-159 48 46 48 80 77 80 F F F <--- 8 160-179 45 45 45 75 75 75 F F F 9 180-199 45 45 45 75 75 75 T F T <--- 10 200-219 41 41 41 68 68 68 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 48 44 48 80 73 80 F F F <--- 13 260-279 45 45 45 75 75 75 F F F 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 48 46 48 80 76 80 F F F <--- 17 340-359 46 46 46 77 77 77 F F F 18 360-379 48 46 48 80 78 80 F F F <--- 19 380-399 46 46 46 78 78 78 T F T <--- 20 400-419 46 46 46 76 76 76 F F F 21 420-439 46 46 46 76 76 76 T F T <--- 22 440-459 46 46 46 76 76 76 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 48 46 48 80 77 80 F F F <--- 27 540-559 41 41 41 68 68 68 F F F 28 560-579 48 41 48 80 68 80 F F F <--- 29 580-599 48 46 48 80 76 80 F F F <--- Max gen: 48, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 81 81 81 F F F 1 020-039 46 46 46 77 77 77 F F F 2 040-059 49 49 49 81 81 81 F F F 3 060-079 49 49 49 81 81 81 F F F 4 080-099 47 47 47 79 79 79 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 48 48 48 80 80 80 F F F 7 140-159 48 48 48 80 80 80 F F F 8 160-179 45 45 45 75 75 75 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 48 48 48 80 80 80 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 43 43 43 72 72 72 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 48 48 48 80 80 80 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 49 49 49 81 81 81 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 49 49 49 81 81 81 F F F 22 440-459 46 46 46 76 76 76 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 48 48 48 80 80 80 F F F Max gen: 49, Max flush: 81 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.078490679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.086628997Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.086958596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.086986252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.086994631Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.08700596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Crucible 0a710e64-5864-4dc6-a037-b26e9764029c has session id: f812c5e7-2091-4a3b-aa6b-8484c3e761f0","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087360635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Upstairs opts: Upstairs UUID: 0a710e64-5864-4dc6-a037-b26e9764029c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087378992Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Crucible stats registered with UUID: 0a710e64-5864-4dc6-a037-b26e9764029c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.08739355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:01.087402649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087550231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087582856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087593715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087717079Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087740916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"0a710e64-5864-4dc6-a037-b26e9764029c active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087755934Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087764893Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"0","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087774582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"1","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087784161Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"2","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.0877938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087803978Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087813857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55279)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087898806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64416)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087924393Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47743)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.087937381Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.088563312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"2","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.088588819Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"1","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.088705074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"0","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.088954682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[0]R flush_numbers[0..12]: [81, 77, 81, 81, 79, 80, 80, 80, 75, 81, 68, 79]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.08897334Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[0]R generation[0..12]: [49, 46, 49, 49, 47, 48, 48, 48, 45, 49, 41, 47]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.088983919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.088994797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[1]R flush_numbers[0..12]: [81, 77, 81, 81, 79, 80, 80, 80, 75, 81, 68, 79]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089005106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[1]R generation[0..12]: [49, 46, 49, 49, 47, 48, 48, 48, 45, 49, 41, 47]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089015435Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089026193Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[2]R flush_numbers[0..12]: [81, 77, 81, 81, 79, 80, 80, 80, 75, 81, 68, 79]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089036582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[2]R generation[0..12]: [49, 46, 49, 49, 47, 48, 48, 48, 45, 49, 41, 47]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089046731Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089057519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089068998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089078747Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089088075Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089097454Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"":"downstairs","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089107003Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089115632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"0","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089125191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"1","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.0891347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"client":"2","":"downstairs","session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089144598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089153727Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"0a710e64-5864-4dc6-a037-b26e9764029c is now active with session: f812c5e7-2091-4a3b-aa6b-8484c3e761f0","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089163826Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089172465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"session_id":"f812c5e7-2091-4a3b-aa6b-8484c3e761f0"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089181544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Activated sub_volume 0a710e64-5864-4dc6-a037-b26e9764029c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089189493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089197532Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.089505163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"task":"crutest"} 01/30 Write block 454 len 40960 data: 8 5 5 5 5 6 4 3 4 6 02/30 Read block 27 len 28672 03/30 Read block 283 len 4096 04/30 Flush 05/30 Write block 390 len 12288 data: 6 6 8 06/30 Flush 07/30 Write block 5 len 32768 data: 2 2 2 2 3 4 5 5 08/30 Read block 318 len 20480 09/30 Write block 141 len 16384 data: 9 9 7 8 10/30 Write block 297 len 4096 data: 4 11/30 Write block 231 len 40960 data: 4 4 4 3 4 5 4 4 2 2 12/30 Write block 322 len 36864 data: 3 3 3 4 3 5 5 5 5 13/30 Read block 185 len 12288 14/30 Read block 320 len 36864 15/30 Flush 16/30 Flush 17/30 Write block 290 len 24576 data: 4 3 4 6 6 7 18/30 Write block 36 len 8192 data: 4 4 19/30 Write block 34 len 4096 data: 5 20/30 Read block 303 len 4096 21/30 Read block 520 len 16384 22/30 Read block 528 len 40960 23/30 Read block 357 len 20480 24/30 Read block 238 len 36864 25/30 Write block 224 len 16384 data: 6 5 5 5 26/30 Read block 139 len 8192 27/30 Read block 93 len 36864 28/30 Write block 69 len 12288 data: 4 4 8 29/30 Read block 187 len 20480 30/30 Read block 249 len 32768 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 19 0 20 Done 23 5 24 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1005)) Some(JobId(1015)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:01.09835963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:01.098519679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1357,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 49 50 84 81 84 F T F <--- 1 020-039 46 46 46 77 77 77 T F T <--- 2 040-059 49 49 49 81 81 81 F F F 3 060-079 49 49 49 81 81 81 T F T <--- 4 080-099 47 47 47 79 79 79 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 48 48 48 80 80 80 F F F 7 140-159 50 48 50 84 80 84 F T F <--- 8 160-179 45 45 45 75 75 75 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 50 47 50 84 79 84 T T T <--- 12 240-259 50 48 50 84 80 84 F T F <--- 13 260-279 45 45 45 75 75 75 F F F 14 280-299 50 43 50 84 72 84 T T T <--- 15 300-319 36 36 36 60 60 60 F F F 16 320-339 50 48 50 84 80 84 F F F <--- 17 340-359 46 46 46 77 77 77 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 50 50 50 83 83 83 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 49 49 49 81 81 81 F F F 22 440-459 50 50 50 82 82 82 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 48 48 48 80 80 80 F F F Max gen: 50, Max flush: 84 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 84 84 84 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 49 49 49 81 81 81 F F F 3 060-079 51 51 51 85 85 85 F F F 4 080-099 47 47 47 79 79 79 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 48 48 48 80 80 80 F F F 7 140-159 50 50 50 84 84 84 F F F 8 160-179 45 45 45 75 75 75 F F F 9 180-199 49 49 49 81 81 81 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 51 51 51 85 85 85 F F F 12 240-259 50 50 50 84 84 84 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 51 51 51 85 85 85 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 50 50 50 84 84 84 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 50 50 50 83 83 83 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 49 49 49 81 81 81 F F F 22 440-459 50 50 50 82 82 82 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 48 48 48 80 80 80 F F F Max gen: 51, Max flush: 85 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.340703179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.348769646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349080376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349105343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349113802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349125361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Crucible 7582680b-7247-493f-a6ab-2a12a1c0fd2f has session id: 88df44f3-188e-404e-9ec3-1384ba89d281","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349479226Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Upstairs opts: Upstairs UUID: 7582680b-7247-493f-a6ab-2a12a1c0fd2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349507382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Crucible stats registered with UUID: 7582680b-7247-493f-a6ab-2a12a1c0fd2f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349521091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:15.349530039Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349653734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349669972Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.34967983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349825832Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349849749Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"7582680b-7247-493f-a6ab-2a12a1c0fd2f active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349862727Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349871506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"0","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349883185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"1","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349892803Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"2","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349902402Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.34991729Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.349927109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"ds_connection connected from Ok(127.0.0.1:59554)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350008859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"ds_connection connected from Ok(127.0.0.1:64404)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350052563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"ds_connection connected from Ok(127.0.0.1:40757)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350109616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350645278Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"1","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350657416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"0","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350718459Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"2","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.350995794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[0]R flush_numbers[0..12]: [84, 85, 81, 85, 79, 80, 80, 84, 75, 81, 68, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.35102145Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[0]R generation[0..12]: [50, 51, 49, 51, 47, 48, 48, 50, 45, 49, 41, 51]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351032439Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351043557Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[1]R flush_numbers[0..12]: [84, 85, 81, 85, 79, 80, 80, 84, 75, 81, 68, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351056036Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[1]R generation[0..12]: [50, 51, 49, 51, 47, 48, 48, 50, 45, 49, 41, 51]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351066465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351077413Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[2]R flush_numbers[0..12]: [84, 85, 81, 85, 79, 80, 80, 84, 75, 81, 68, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351087632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[2]R generation[0..12]: [50, 51, 49, 51, 47, 48, 48, 50, 45, 49, 41, 51]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351097901Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351108739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351118058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351127687Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351136936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351146154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"":"downstairs","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351155643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351164362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"0","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351174191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"1","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.35118382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"client":"2","":"downstairs","session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351193608Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351203837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"7582680b-7247-493f-a6ab-2a12a1c0fd2f is now active with session: 88df44f3-188e-404e-9ec3-1384ba89d281","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351213736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351222435Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"session_id":"88df44f3-188e-404e-9ec3-1384ba89d281"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351231444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Activated sub_volume 7582680b-7247-493f-a6ab-2a12a1c0fd2f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351239573Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351247632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.351559872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"task":"crutest"} 01/30 Write block 146 len 4096 data: 9 02/30 Write block 48 len 24576 data: 5 6 5 6 6 6 03/30 Read block 441 len 16384 04/30 Write block 128 len 40960 data: 5 7 6 9 9 7 7 6 5 4 05/30 Read block 260 len 8192 06/30 Write block 59 len 8192 data: 5 4 07/30 Read block 169 len 8192 08/30 Write block 97 len 8192 data: 3 4 09/30 Read block 354 len 4096 10/30 Read block 153 len 12288 11/30 Write block 192 len 20480 data: 6 6 5 4 4 12/30 Write block 152 len 24576 data: 7 7 6 4 6 7 13/30 Write block 439 len 8192 data: 4 4 14/30 Read block 571 len 20480 15/30 Read block 541 len 20480 16/30 Read block 316 len 24576 17/30 Write block 11 len 16384 data: 6 6 4 4 18/30 Write block 238 len 36864 data: 5 3 3 2 3 3 2 3 2 19/30 Read block 80 len 16384 20/30 Write block 168 len 28672 data: 9 10 10 9 9 8 8 21/30 Read block 78 len 16384 22/30 Write block 175 len 12288 data: 9 8 7 23/30 Write block 583 len 4096 data: 6 24/30 Read block 561 len 4096 25/30 Read block 430 len 4096 26/30 Write block 205 len 32768 data: 5 5 6 7 5 5 5 6 27/30 Read block 114 len 20480 28/30 Write block 63 len 20480 data: 5 4 4 4 4 29/30 Read block 124 len 36864 30/30 Write block 156 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 9 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 30 33 Done 29 28 0 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:15.359240468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:15.359410506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1366,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 84 84 84 T T F <--- 1 020-039 51 51 51 85 85 85 F F F 2 040-059 49 49 49 81 81 81 T T F <--- 3 060-079 51 51 51 85 85 85 T T F <--- 4 080-099 47 47 47 79 79 79 T T F <--- 5 100-119 48 48 48 80 80 80 F F F 6 120-139 48 48 48 80 80 80 T T F <--- 7 140-159 50 50 50 84 84 84 T T F <--- 8 160-179 45 45 45 75 75 75 T T F <--- 9 180-199 49 49 49 81 81 81 T T F <--- 10 200-219 41 41 41 68 68 68 T T F <--- 11 220-239 51 51 51 85 85 85 T T F <--- 12 240-259 50 50 50 84 84 84 T T F <--- 13 260-279 45 45 45 75 75 75 F F F 14 280-299 51 51 51 85 85 85 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 50 50 50 84 84 84 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 50 50 50 83 83 83 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 49 49 49 81 81 81 T T F <--- 22 440-459 50 50 50 82 82 82 T T F <--- 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 48 48 48 80 80 80 T T F <--- Max gen: 51, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 86 86 86 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 52 52 52 86 86 86 F F F 3 060-079 52 52 52 86 86 86 F F F 4 080-099 52 52 52 86 86 86 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 52 52 52 86 86 86 F F F 7 140-159 52 52 52 86 86 86 F F F 8 160-179 52 52 52 86 86 86 F F F 9 180-199 52 52 52 86 86 86 F F F 10 200-219 52 52 52 86 86 86 F F F 11 220-239 52 52 52 86 86 86 F F F 12 240-259 52 52 52 86 86 86 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 51 51 51 85 85 85 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 50 50 50 84 84 84 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 50 50 50 83 83 83 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 52 52 52 86 86 86 F F F 22 440-459 52 52 52 86 86 86 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 48 48 48 80 80 80 F F F 27 540-559 41 41 41 68 68 68 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 52 52 52 86 86 86 F F F Max gen: 52, Max flush: 86 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.613648187Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.621605458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.62190199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.621931067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.621939575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.621951044Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Crucible 7e0891e3-5101-4932-aab4-49c2f22ad8ac has session id: cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622304979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Upstairs opts: Upstairs UUID: 7e0891e3-5101-4932-aab4-49c2f22ad8ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.6223745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Crucible stats registered with UUID: 7e0891e3-5101-4932-aab4-49c2f22ad8ac","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622394598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:29.622403647Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622525371Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622541729Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622551798Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622709628Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622733295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"7e0891e3-5101-4932-aab4-49c2f22ad8ac active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622745673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622754512Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"0","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622764191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"1","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.62277363Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"2","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622783298Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622793157Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622843251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37533)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622909992Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:65259)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622973764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57309)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.622986993Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623533703Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"0","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623548741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"2","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623615423Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"1","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.62387592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[0]R flush_numbers[0..12]: [86, 85, 86, 86, 86, 80, 86, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623894448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[0]R generation[0..12]: [52, 51, 52, 52, 52, 48, 52, 52, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623905036Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623915915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[1]R flush_numbers[0..12]: [86, 85, 86, 86, 86, 80, 86, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623926223Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[1]R generation[0..12]: [52, 51, 52, 52, 52, 48, 52, 52, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623936342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623947131Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[2]R flush_numbers[0..12]: [86, 85, 86, 86, 86, 80, 86, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.62395752Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[2]R generation[0..12]: [52, 51, 52, 52, 52, 48, 52, 52, 52, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623967688Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623978437Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.623989805Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624003504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624012723Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624022001Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"":"downstairs","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.62403169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624040669Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"0","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624050328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"1","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624059737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"client":"2","":"downstairs","session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624069575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624078564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"7e0891e3-5101-4932-aab4-49c2f22ad8ac is now active with session: cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624088403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624097152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"session_id":"cb4d9d32-1fe9-4c56-83df-e65b6e9dd2d5"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624106101Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Activated sub_volume 7e0891e3-5101-4932-aab4-49c2f22ad8ac","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.62411406Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624122089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.624405523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"task":"crutest"} 01/30 Write block 563 len 28672 data: 5 3 3 3 3 3 3 02/30 Read block 394 len 20480 03/30 Read block 560 len 24576 04/30 Write block 539 len 12288 data: 12 11 10 05/30 Read block 3 len 28672 06/30 Read block 198 len 36864 07/30 Write block 337 len 4096 data: 5 08/30 Read block 288 len 28672 09/30 Write block 97 len 4096 data: 4 10/30 Read block 168 len 36864 11/30 Write block 588 len 36864 data: 5 4 4 5 5 5 5 5 5 12/30 Write block 50 len 8192 data: 6 7 13/30 Read block 264 len 32768 14/30 Write block 238 len 12288 data: 6 4 4 15/30 Read block 203 len 24576 16/30 Write block 574 len 36864 data: 6 6 6 6 5 6 7 7 6 17/30 Read block 573 len 28672 18/30 Read block 2 len 40960 19/30 Flush 20/30 Write block 296 len 16384 data: 7 5 4 4 21/30 Read block 112 len 32768 22/30 Read block 264 len 36864 23/30 Write block 176 len 32768 data: 9 8 4 4 6 5 5 8 24/30 Read block 102 len 8192 25/30 Read block 436 len 40960 26/30 Write block 12 len 16384 data: 7 5 5 4 27/30 Read block 249 len 32768 28/30 Read block 571 len 28672 29/30 Write block 339 len 36864 data: 5 5 6 4 5 3 4 4 3 30/30 Read block 61 len 40960 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:29.634302178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:29.634475136Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1375,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 86 86 86 T F T <--- 1 020-039 51 51 51 85 85 85 F F F 2 040-059 54 52 54 87 86 87 F F F <--- 3 060-079 52 52 52 86 86 86 F F F 4 080-099 54 52 54 87 86 87 F F F <--- 5 100-119 48 48 48 80 80 80 F F F 6 120-139 52 52 52 86 86 86 F F F 7 140-159 52 52 52 86 86 86 F F F 8 160-179 52 52 52 86 86 86 T F T <--- 9 180-199 52 52 52 86 86 86 T F T <--- 10 200-219 52 52 52 86 86 86 F F F 11 220-239 54 52 54 87 86 87 F F F <--- 12 240-259 54 52 54 87 86 87 F F F <--- 13 260-279 45 45 45 75 75 75 F F F 14 280-299 51 51 51 85 85 85 T F T <--- 15 300-319 36 36 36 60 60 60 F F F 16 320-339 54 50 54 87 84 87 T F T <--- 17 340-359 46 46 46 77 77 77 T F T <--- 18 360-379 48 48 48 80 80 80 F F F 19 380-399 50 50 50 83 83 83 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 52 52 52 86 86 86 F F F 22 440-459 52 52 52 86 86 86 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 54 48 54 87 80 87 F F F <--- 27 540-559 54 41 54 87 68 87 F F F <--- 28 560-579 54 48 54 87 80 87 F F F <--- 29 580-599 54 52 54 87 86 87 F F F <--- Max gen: 54, Max flush: 87 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 88 88 88 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 54 54 54 87 87 87 F F F 3 060-079 52 52 52 86 86 86 F F F 4 080-099 54 54 54 87 87 87 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 52 52 52 86 86 86 F F F 7 140-159 52 52 52 86 86 86 F F F 8 160-179 55 55 55 88 88 88 F F F 9 180-199 55 55 55 88 88 88 F F F 10 200-219 52 52 52 86 86 86 F F F 11 220-239 54 54 54 87 87 87 F F F 12 240-259 54 54 54 87 87 87 F F F 13 260-279 45 45 45 75 75 75 F F F 14 280-299 55 55 55 88 88 88 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 55 55 55 88 88 88 F F F 17 340-359 55 55 55 88 88 88 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 50 50 50 83 83 83 F F F 20 400-419 46 46 46 76 76 76 F F F 21 420-439 52 52 52 86 86 86 F F F 22 440-459 52 52 52 86 86 86 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 54 54 54 87 87 87 F F F 27 540-559 54 54 54 87 87 87 F F F 28 560-579 54 54 54 87 87 87 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 55, Max flush: 88 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.889432095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.897370568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.897655092Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.897679579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.897687858Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.897699056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Crucible d5d42991-cdd2-4f89-a87b-34cc5a4f38fb has session id: 80c436f8-73a4-48da-aaac-081e9cb26d62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898108784Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Upstairs opts: Upstairs UUID: d5d42991-cdd2-4f89-a87b-34cc5a4f38fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898137431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Crucible stats registered with UUID: d5d42991-cdd2-4f89-a87b-34cc5a4f38fb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898148829Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:43.898158068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898286702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898316618Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898327326Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898464209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898488166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"d5d42991-cdd2-4f89-a87b-34cc5a4f38fb active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898501164Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898512223Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"0","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898521842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"1","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898531351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"2","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898540759Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898550498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898560247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"ds_connection connected from Ok(127.0.0.1:49602)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898644386Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"ds_connection connected from Ok(127.0.0.1:43264)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898669373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"ds_connection connected from Ok(127.0.0.1:47536)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.898720657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899376073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"1","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899391751Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"0","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899429347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"2","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899695943Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[0]R flush_numbers[0..12]: [88, 85, 87, 86, 87, 80, 86, 86, 88, 88, 86, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899714231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[0]R generation[0..12]: [55, 51, 54, 52, 54, 48, 52, 52, 55, 55, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899727209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899738277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[1]R flush_numbers[0..12]: [88, 85, 87, 86, 87, 80, 86, 86, 88, 88, 86, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899748626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[1]R generation[0..12]: [55, 51, 54, 52, 54, 48, 52, 52, 55, 55, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899758875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899769683Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[2]R flush_numbers[0..12]: [88, 85, 87, 86, 87, 80, 86, 86, 88, 88, 86, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899780002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[2]R generation[0..12]: [55, 51, 54, 52, 54, 48, 52, 52, 55, 55, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899790361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.89980114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899810898Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899820547Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899829706Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899839095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"":"downstairs","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899848813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899857652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"0","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899867291Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"1","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.89987802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"client":"2","":"downstairs","session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899887869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899897057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"d5d42991-cdd2-4f89-a87b-34cc5a4f38fb is now active with session: 80c436f8-73a4-48da-aaac-081e9cb26d62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899906966Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899915675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"session_id":"80c436f8-73a4-48da-aaac-081e9cb26d62"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899924564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Activated sub_volume d5d42991-cdd2-4f89-a87b-34cc5a4f38fb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899932423Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.899940422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.900242734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"task":"crutest"} 01/30 Write block 8 len 4096 data: 3 02/30 Read block 90 len 40960 03/30 Write block 359 len 36864 data: 5 4 4 4 5 6 5 5 6 04/30 Write block 148 len 32768 data: 10 10 9 7 8 8 7 5 05/30 Write block 402 len 32768 data: 5 5 5 4 4 4 3 2 06/30 Write block 273 len 40960 data: 3 3 4 5 6 6 6 7 7 7 07/30 Read block 163 len 24576 08/30 Read block 101 len 4096 09/30 Write block 393 len 8192 data: 8 8 10/30 Read block 485 len 12288 11/30 Write block 205 len 20480 data: 6 6 7 8 6 12/30 Write block 193 len 16384 data: 7 6 5 5 13/30 Read block 505 len 16384 14/30 Write block 435 len 24576 data: 4 4 4 4 5 5 15/30 Flush 16/30 Write block 289 len 24576 data: 5 5 4 5 7 7 17/30 Flush 18/30 Write block 46 len 24576 data: 6 6 6 7 7 8 19/30 Write block 539 len 8192 data: 13 12 20/30 Flush 21/30 Write block 244 len 32768 data: 3 4 3 3 6 5 3 5 22/30 Read block 485 len 12288 23/30 Write block 540 len 20480 data: 13 11 9 7 7 24/30 Read block 296 len 16384 25/30 Read block 390 len 4096 26/30 Write block 374 len 36864 data: 5 7 7 7 7 6 6 5 4 27/30 Write block 124 len 40960 data: 3 4 4 5 6 8 7 10 10 8 28/30 Read block 381 len 8192 29/30 Read block 438 len 32768 30/30 Read block 321 len 16384 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 10 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:43.909010112Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:43.90917779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1384,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 55 56 89 88 89 F T F <--- 1 020-039 51 51 51 85 85 85 F F F 2 040-059 56 54 56 91 87 91 F F F <--- 3 060-079 52 52 52 86 86 86 F F F 4 080-099 54 54 54 87 87 87 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 52 52 52 86 86 86 T F T <--- 7 140-159 56 52 56 89 86 89 F F F <--- 8 160-179 55 55 55 88 88 88 F F F 9 180-199 56 55 56 89 88 89 F F F <--- 10 200-219 56 52 56 89 86 89 F F F <--- 11 220-239 54 54 54 87 87 87 F F F 12 240-259 54 54 54 87 87 87 T F T <--- 13 260-279 56 45 56 89 75 89 F F F <--- 14 280-299 56 55 56 90 88 90 F F F <--- 15 300-319 36 36 36 60 60 60 F F F 16 320-339 55 55 55 88 88 88 F F F 17 340-359 56 55 56 89 88 89 F F F <--- 18 360-379 56 48 56 89 80 89 T F T <--- 19 380-399 56 50 56 89 83 89 T F T <--- 20 400-419 56 46 56 89 76 89 F F F <--- 21 420-439 56 52 56 89 86 89 F F F <--- 22 440-459 56 52 56 89 86 89 F F F <--- 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 56 54 56 91 87 91 F F F <--- 27 540-559 56 54 56 91 87 91 T F T <--- 28 560-579 54 54 54 87 87 87 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 56, Max flush: 91 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 89 89 89 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 56 56 56 91 91 91 F F F 3 060-079 52 52 52 86 86 86 F F F 4 080-099 54 54 54 87 87 87 F F F 5 100-119 48 48 48 80 80 80 F F F 6 120-139 57 57 57 92 92 92 F F F 7 140-159 56 56 56 89 89 89 F F F 8 160-179 55 55 55 88 88 88 F F F 9 180-199 56 56 56 89 89 89 F F F 10 200-219 56 56 56 89 89 89 F F F 11 220-239 54 54 54 87 87 87 F F F 12 240-259 57 57 57 92 92 92 F F F 13 260-279 56 56 56 89 89 89 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 55 55 55 88 88 88 F F F 17 340-359 56 56 56 89 89 89 F F F 18 360-379 57 57 57 92 92 92 F F F 19 380-399 57 57 57 92 92 92 F F F 20 400-419 56 56 56 89 89 89 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 56 56 56 89 89 89 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 46 46 46 76 76 76 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 56 56 56 91 91 91 F F F 27 540-559 57 57 57 92 92 92 F F F 28 560-579 54 54 54 87 87 87 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 57, Max flush: 92 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.169630213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.177659505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.177950028Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.177976434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.177990493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178001991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Crucible 2317c4ab-3bf7-43e7-881d-dec105478adc has session id: 3666fe3f-2fcb-46c3-a015-d6fa2ae9a125","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178350667Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Upstairs opts: Upstairs UUID: 2317c4ab-3bf7-43e7-881d-dec105478adc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178380983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Crucible stats registered with UUID: 2317c4ab-3bf7-43e7-881d-dec105478adc","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178391542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:58.178400581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178541623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178572249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178583187Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178699663Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178716371Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"2317c4ab-3bf7-43e7-881d-dec105478adc active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178733108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178741877Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"0","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178751486Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"1","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178761015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"2","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178770574Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178780562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178790541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"ds_connection connected from Ok(127.0.0.1:46445)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.17887923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"ds_connection connected from Ok(127.0.0.1:61928)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178911016Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"ds_connection connected from Ok(127.0.0.1:58131)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.178922684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.17958808Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"1","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.179613757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"0","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.179636004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"2","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.179935036Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[0]R flush_numbers[0..12]: [89, 85, 91, 86, 87, 80, 92, 89, 88, 89, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.179969262Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[0]R generation[0..12]: [56, 51, 56, 52, 54, 48, 57, 56, 55, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.17998022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.179991199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[1]R flush_numbers[0..12]: [89, 85, 91, 86, 87, 80, 92, 89, 88, 89, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180001458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[1]R generation[0..12]: [56, 51, 56, 52, 54, 48, 57, 56, 55, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180011616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180022485Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[2]R flush_numbers[0..12]: [89, 85, 91, 86, 87, 80, 92, 89, 88, 89, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180032794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[2]R generation[0..12]: [56, 51, 56, 52, 54, 48, 57, 56, 55, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180045682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180056561Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180065869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180075638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180084887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180094326Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"":"downstairs","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180103865Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180112663Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"0","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180122292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"1","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180131861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"client":"2","":"downstairs","session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.18014152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180150589Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"2317c4ab-3bf7-43e7-881d-dec105478adc is now active with session: 3666fe3f-2fcb-46c3-a015-d6fa2ae9a125","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180160467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180169096Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"session_id":"3666fe3f-2fcb-46c3-a015-d6fa2ae9a125"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180183154Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Activated sub_volume 2317c4ab-3bf7-43e7-881d-dec105478adc","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180196963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180206252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.180426954Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"task":"crutest"} 01/30 Write block 307 len 36864 data: 7 6 5 5 5 3 2 2 2 02/30 Read block 461 len 12288 03/30 Write block 362 len 12288 data: 5 6 7 04/30 Write block 379 len 4096 data: 7 05/30 Read block 295 len 20480 06/30 Read block 394 len 24576 07/30 Write block 345 len 12288 data: 5 5 4 08/30 Read block 203 len 36864 09/30 Flush 10/30 Write block 383 len 16384 data: 4 3 2 2 11/30 Write block 269 len 32768 data: 3 3 3 3 4 4 5 6 12/30 Write block 169 len 24576 data: 11 11 10 10 9 9 13/30 Read block 287 len 36864 14/30 Read block 253 len 16384 15/30 Write block 552 len 16384 data: 4 4 4 6 16/30 Read block 452 len 36864 17/30 Write block 565 len 12288 data: 4 4 4 18/30 Read block 365 len 12288 19/30 Read block 537 len 12288 20/30 Write block 487 len 12288 data: 4 2 2 21/30 Flush 22/30 Write block 11 len 32768 data: 7 8 6 6 5 4 4 4 23/30 Read block 139 len 20480 24/30 Read block 267 len 32768 25/30 Read block 372 len 32768 26/30 Write block 95 len 8192 data: 4 4 27/30 Write block 528 len 16384 data: 5 5 7 7 28/30 Write block 483 len 12288 data: 5 5 6 29/30 Write block 446 len 24576 data: 6 7 6 6 6 6 30/30 Write block 98 len 40960 data: 5 4 5 8 7 7 6 6 6 8 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 3 Sent Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 27 2 32 Done 27 3 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:51:58.188934785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:51:58.189108453Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1393,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 89 89 89 T F T <--- 1 020-039 51 51 51 85 85 85 F F F 2 040-059 56 56 56 91 91 91 F F F 3 060-079 52 52 52 86 86 86 F F F 4 080-099 54 54 54 87 87 87 T F T <--- 5 100-119 48 48 48 80 80 80 T F T <--- 6 120-139 57 57 57 92 92 92 F F F 7 140-159 56 56 56 89 89 89 F F F 8 160-179 58 55 58 94 88 94 F F F <--- 9 180-199 56 56 56 89 89 89 F F F 10 200-219 56 56 56 89 89 89 F F F 11 220-239 54 54 54 87 87 87 F F F 12 240-259 57 57 57 92 92 92 F F F 13 260-279 58 56 58 94 89 94 F F F <--- 14 280-299 56 56 56 90 90 90 F F F 15 300-319 58 36 58 93 60 93 F T F <--- 16 320-339 55 55 55 88 88 88 F F F 17 340-359 58 56 58 93 89 93 F F F <--- 18 360-379 58 57 58 93 92 93 F T F <--- 19 380-399 58 57 58 94 92 94 F F F <--- 20 400-419 56 56 56 89 89 89 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 56 56 56 89 89 89 T F T <--- 23 460-479 50 50 50 82 82 82 F F F 24 480-499 58 46 58 94 76 94 T F T <--- 25 500-519 46 46 46 77 77 77 F F F 26 520-539 56 56 56 91 91 91 T F T <--- 27 540-559 58 57 58 94 92 94 F F F <--- 28 560-579 58 54 58 94 87 94 F F F <--- 29 580-599 54 54 54 87 87 87 F F F Max gen: 58, Max flush: 94 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 56 56 56 91 91 91 F F F 3 060-079 52 52 52 86 86 86 F F F 4 080-099 59 59 59 95 95 95 F F F 5 100-119 59 59 59 95 95 95 F F F 6 120-139 57 57 57 92 92 92 F F F 7 140-159 56 56 56 89 89 89 F F F 8 160-179 58 58 58 94 94 94 F F F 9 180-199 56 56 56 89 89 89 F F F 10 200-219 56 56 56 89 89 89 F F F 11 220-239 54 54 54 87 87 87 F F F 12 240-259 57 57 57 92 92 92 F F F 13 260-279 58 58 58 94 94 94 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 58 58 58 93 93 93 F F F 16 320-339 55 55 55 88 88 88 F F F 17 340-359 58 58 58 93 93 93 F F F 18 360-379 58 58 58 93 93 93 F F F 19 380-399 58 58 58 94 94 94 F F F 20 400-419 56 56 56 89 89 89 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 59 59 59 95 95 95 F F F 23 460-479 50 50 50 82 82 82 F F F 24 480-499 59 59 59 95 95 95 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 59 59 59 95 95 95 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 94 94 94 F F F 29 580-599 54 54 54 87 87 87 F F F Max gen: 59, Max flush: 95 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.445621935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:12.452680659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.452726764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.459520652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.45977622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.459804836Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.459813335Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.459824693Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Crucible 26c610a3-9ed8-4529-adec-13e1961d64b3 has session id: 1bfef33d-983b-4dce-a799-6837475c6e48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460153222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Upstairs opts: Upstairs UUID: 26c610a3-9ed8-4529-adec-13e1961d64b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460192837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Crucible stats registered with UUID: 26c610a3-9ed8-4529-adec-13e1961d64b3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460206285Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:12.460215214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460358946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460391342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.46040261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460497608Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460521715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"26c610a3-9ed8-4529-adec-13e1961d64b3 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460537863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460546622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"0","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460556281Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"1","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460565749Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"2","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460575208Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460585077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460594896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:62134)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460674476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:38204)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460771063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"ds_connection connected from Ok(127.0.0.1:46378)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.460785302Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461366528Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"1","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461388855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"0","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461434159Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"2","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.46174603Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[0]R flush_numbers[0..12]: [95, 85, 91, 86, 95, 95, 92, 89, 94, 89, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461774046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[0]R generation[0..12]: [59, 51, 56, 52, 59, 59, 57, 56, 58, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461784855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461795863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[1]R flush_numbers[0..12]: [95, 85, 91, 86, 95, 95, 92, 89, 94, 89, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461806122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[1]R generation[0..12]: [59, 51, 56, 52, 59, 59, 57, 56, 58, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461816421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461827109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[2]R flush_numbers[0..12]: [95, 85, 91, 86, 95, 95, 92, 89, 94, 89, 89, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461837478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[2]R generation[0..12]: [59, 51, 56, 52, 59, 59, 57, 56, 58, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461847547Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461858285Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461867564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461877103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461886382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461897321Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"":"downstairs","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461906979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461915688Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"0","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461925287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"1","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461934746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"client":"2","":"downstairs","session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461944565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461953653Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"26c610a3-9ed8-4529-adec-13e1961d64b3 is now active with session: 1bfef33d-983b-4dce-a799-6837475c6e48","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461963452Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461972201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"session_id":"1bfef33d-983b-4dce-a799-6837475c6e48"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.46198134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Activated sub_volume 26c610a3-9ed8-4529-adec-13e1961d64b3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.461989239Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.462007927Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.462249696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} 01/30 Write block 552 len 8192 data: 5 5 02/30 Read block 328 len 28672 03/30 Write block 314 len 28672 data: 3 3 2 2 3 3 3 04/30 Read block 14 len 12288 05/30 Write block 152 len 12288 data: 9 9 8 06/30 Read block 256 len 20480 07/30 Write block 197 len 40960 data: 3 4 5 5 5 5 6 6 7 7 08/30 Write block 437 len 36864 data: 5 5 6 6 4 3 4 4 5 09/30 Read block 259 len 12288 10/30 Read block 91 len 24576 11/30 Write block 542 len 24576 data: 10 8 8 6 4 4 12/30 Write block 325 len 4096 data: 5 13/30 Read block 549 len 20480 14/30 Read block 342 len 16384 15/30 Read block 495 len 24576 16/30 Write block 109 len 40960 data: 9 7 6 5 5 4 4 3 3 4 17/30 Write block 590 len 12288 data: 5 6 6 18/30 Read block 329 len 36864 19/30 Write block 59 len 16384 data: 6 5 4 5 20/30 Write block 184 len 40960 data: 6 5 6 5 5 6 6 6 7 8 21/30 Read block 183 len 28672 22/30 Write block 505 len 36864 data: 6 4 4 6 4 4 4 5 4 23/30 Read block 503 len 12288 24/30 Write block 535 len 32768 data: 8 8 7 9 14 14 12 11 25/30 Flush 26/30 Write block 519 len 36864 data: 4 4 4 2 3 3 5 5 5 27/30 Write block 453 len 12288 data: 8 9 6 28/30 Write block 462 len 28672 data: 5 7 5 5 4 3 5 29/30 Read block 478 len 16384 30/30 Write block 161 len 32768 data: 6 6 8 9 6 5 7 10 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:12.471629226Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:12.471781967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1402,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 56 60 60 91 96 96 F F F <--- 3 060-079 52 60 60 86 96 96 F F F <--- 4 080-099 59 59 59 95 95 95 F F F 5 100-119 59 60 60 95 96 96 F F F <--- 6 120-139 57 57 57 92 92 92 F F F 7 140-159 56 60 60 89 96 96 F F F <--- 8 160-179 58 58 58 94 94 94 F T T <--- 9 180-199 56 60 60 89 96 96 F F F <--- 10 200-219 56 60 60 89 96 96 F F F <--- 11 220-239 54 54 54 87 87 87 F F F 12 240-259 57 57 57 92 92 92 F F F 13 260-279 58 58 58 94 94 94 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 58 60 60 93 96 96 T F F <--- 16 320-339 55 60 60 88 96 96 T F F <--- 17 340-359 58 58 58 93 93 93 F F F 18 360-379 58 58 58 93 93 93 F F F 19 380-399 58 58 58 94 94 94 F F F 20 400-419 56 56 56 89 89 89 F F F 21 420-439 56 60 60 89 96 96 F F F <--- 22 440-459 59 60 60 95 96 96 F T T <--- 23 460-479 50 50 50 82 82 82 F T T <--- 24 480-499 59 59 59 95 95 95 F F F 25 500-519 46 60 60 77 96 96 F T T <--- 26 520-539 59 60 60 95 96 96 F T T <--- 27 540-559 58 60 60 94 96 96 T F F <--- 28 560-579 58 58 58 94 94 94 F F F 29 580-599 54 60 60 87 96 96 F F F <--- Max gen: 60, Max flush: 96 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 51 51 51 85 85 85 F F F 2 040-059 60 60 60 96 96 96 F F F 3 060-079 60 60 60 96 96 96 F F F 4 080-099 59 59 59 95 95 95 F F F 5 100-119 60 60 60 96 96 96 F F F 6 120-139 57 57 57 92 92 92 F F F 7 140-159 60 60 60 96 96 96 F F F 8 160-179 61 61 61 97 97 97 F F F 9 180-199 60 60 60 96 96 96 F F F 10 200-219 60 60 60 96 96 96 F F F 11 220-239 54 54 54 87 87 87 F F F 12 240-259 57 57 57 92 92 92 F F F 13 260-279 58 58 58 94 94 94 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 60 60 60 96 96 96 F F F 16 320-339 60 60 60 96 96 96 F F F 17 340-359 58 58 58 93 93 93 F F F 18 360-379 58 58 58 93 93 93 F F F 19 380-399 58 58 58 94 94 94 F F F 20 400-419 56 56 56 89 89 89 F F F 21 420-439 60 60 60 96 96 96 F F F 22 440-459 61 61 61 97 97 97 F F F 23 460-479 61 61 61 97 97 97 F F F 24 480-499 59 59 59 95 95 95 F F F 25 500-519 61 61 61 97 97 97 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 58 58 58 94 94 94 F F F 29 580-599 60 60 60 96 96 96 F F F Max gen: 61, Max flush: 97 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.729507075Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:26.736559161Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.736613174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.743313944Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.743571522Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.743595639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.743603728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.743615156Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Crucible fada8e1f-1fdb-4f18-b084-a11177078687 has session id: e2f6907c-ad8a-445b-bb24-f6a448108c2f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.743968501Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Upstairs opts: Upstairs UUID: fada8e1f-1fdb-4f18-b084-a11177078687, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744031563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Crucible stats registered with UUID: fada8e1f-1fdb-4f18-b084-a11177078687","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744042022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:26.744076558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744227078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744257225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744268073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744391868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744403666Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"fada8e1f-1fdb-4f18-b084-a11177078687 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744426863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744438052Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"0","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744447651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"1","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744457279Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"2","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744466588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744476447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744486346Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50386)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744503073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37632)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.744571605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54950)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.74468716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.74515899Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"2","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745182407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"1","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745274846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"0","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745568448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[0]R flush_numbers[0..12]: [95, 85, 96, 96, 95, 96, 92, 96, 97, 96, 96, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745589656Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[0]R generation[0..12]: [59, 51, 60, 60, 59, 60, 57, 60, 61, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745601104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745614743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[1]R flush_numbers[0..12]: [95, 85, 96, 96, 95, 96, 92, 96, 97, 96, 96, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745625421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[1]R generation[0..12]: [59, 51, 60, 60, 59, 60, 57, 60, 61, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.74563574Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745646798Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[2]R flush_numbers[0..12]: [95, 85, 96, 96, 95, 96, 92, 96, 97, 96, 96, 87]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745657217Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[2]R generation[0..12]: [59, 51, 60, 60, 59, 60, 57, 60, 61, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745667516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745678264Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745687623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745697302Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745706581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.74571613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"":"downstairs","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745725738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745734547Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"0","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745744266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"1","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745755035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"client":"2","":"downstairs","session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745764903Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745774052Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"fada8e1f-1fdb-4f18-b084-a11177078687 is now active with session: e2f6907c-ad8a-445b-bb24-f6a448108c2f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745783981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.74579269Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"session_id":"e2f6907c-ad8a-445b-bb24-f6a448108c2f"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745801809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Activated sub_volume fada8e1f-1fdb-4f18-b084-a11177078687","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745809828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.745817827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.746101641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} 01/30 Flush 02/30 Write block 264 len 28672 data: 6 5 4 3 3 4 4 03/30 Read block 298 len 24576 04/30 Write block 139 len 4096 data: 8 05/30 Write block 330 len 40960 data: 6 3 4 4 5 6 6 6 7 6 06/30 Read block 528 len 28672 07/30 Read block 13 len 12288 08/30 Write block 40 len 24576 data: 7 7 8 7 7 5 09/30 Write block 302 len 28672 data: 4 6 6 6 6 8 7 10/30 Read block 53 len 4096 11/30 Flush 12/30 Write block 57 len 20480 data: 6 7 7 6 5 13/30 Write block 342 len 20480 data: 5 6 4 6 6 14/30 Read block 98 len 16384 15/30 Write block 144 len 24576 data: 9 7 10 9 11 11 16/30 Read block 419 len 28672 17/30 Read block 118 len 4096 18/30 Flush 19/30 Write block 235 len 32768 data: 5 6 5 7 5 5 3 4 20/30 Read block 377 len 24576 21/30 Read block 546 len 40960 22/30 Read block 209 len 32768 23/30 Read block 110 len 40960 24/30 Read block 41 len 40960 25/30 Write block 124 len 16384 data: 4 5 5 6 26/30 Read block 101 len 40960 27/30 Write block 25 len 36864 data: 3 3 3 3 4 4 4 4 5 28/30 Write block 403 len 36864 data: 6 6 5 5 5 4 3 2 2 29/30 Write block 98 len 20480 data: 6 5 6 9 8 30/30 Read block 99 len 20480 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 1 29 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:26.755755646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:26.755920645Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1411,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 51 51 51 85 85 85 F T T <--- 2 040-059 60 62 62 96 100 100 F F F <--- 3 060-079 60 62 62 96 100 100 F F F <--- 4 080-099 59 59 59 95 95 95 F T T <--- 5 100-119 60 60 60 96 96 96 F T T <--- 6 120-139 57 62 62 92 99 99 F T T <--- 7 140-159 60 62 62 96 100 100 F F F <--- 8 160-179 61 61 61 97 97 97 F F F 9 180-199 60 60 60 96 96 96 F F F 10 200-219 60 60 60 96 96 96 F F F 11 220-239 54 54 54 87 87 87 F T T <--- 12 240-259 57 57 57 92 92 92 F T T <--- 13 260-279 58 62 62 94 99 99 T F F <--- 14 280-299 56 56 56 90 90 90 F F F 15 300-319 60 62 62 96 99 99 F F F <--- 16 320-339 60 62 62 96 99 99 F F F <--- 17 340-359 58 62 62 93 100 100 F F F <--- 18 360-379 58 58 58 93 93 93 F F F 19 380-399 58 58 58 94 94 94 F F F 20 400-419 56 56 56 89 89 89 F T T <--- 21 420-439 60 60 60 96 96 96 F F F 22 440-459 61 61 61 97 97 97 F F F 23 460-479 61 61 61 97 97 97 F F F 24 480-499 59 59 59 95 95 95 F F F 25 500-519 61 61 61 97 97 97 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 58 58 58 94 94 94 F F F 29 580-599 60 60 60 96 96 96 F F F Max gen: 62, Max flush: 100 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 63 63 63 101 101 101 F F F 2 040-059 62 62 62 100 100 100 F F F 3 060-079 62 62 62 100 100 100 F F F 4 080-099 63 63 63 101 101 101 F F F 5 100-119 63 63 63 101 101 101 F F F 6 120-139 63 63 63 101 101 101 F F F 7 140-159 62 62 62 100 100 100 F F F 8 160-179 61 61 61 97 97 97 F F F 9 180-199 60 60 60 96 96 96 F F F 10 200-219 60 60 60 96 96 96 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 63 63 63 101 101 101 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 62 62 62 99 99 99 F F F 16 320-339 62 62 62 99 99 99 F F F 17 340-359 62 62 62 100 100 100 F F F 18 360-379 58 58 58 93 93 93 F F F 19 380-399 58 58 58 94 94 94 F F F 20 400-419 63 63 63 101 101 101 F F F 21 420-439 60 60 60 96 96 96 F F F 22 440-459 61 61 61 97 97 97 F F F 23 460-479 61 61 61 97 97 97 F F F 24 480-499 59 59 59 95 95 95 F F F 25 500-519 61 61 61 97 97 97 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 58 58 58 94 94 94 F F F 29 580-599 60 60 60 96 96 96 F F F Max gen: 63, Max flush: 101 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.011140851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019197979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019484793Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.01950955Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019518309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019529837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Crucible 1cfeba8f-ce01-47b7-b0cf-e623917c8f91 has session id: a37de63f-ac93-4097-ac36-ff75919e162d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019909209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Upstairs opts: Upstairs UUID: 1cfeba8f-ce01-47b7-b0cf-e623917c8f91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019939095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Crucible stats registered with UUID: 1cfeba8f-ce01-47b7-b0cf-e623917c8f91","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.019952853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:41.019961922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020106384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020122722Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020132721Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020286441Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020302929Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"1cfeba8f-ce01-47b7-b0cf-e623917c8f91 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020316087Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020324846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"0","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020334505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"1","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020344054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"2","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020353643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020363421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.02037328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33264)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020440072Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64780)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.02045294Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33645)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.020551348Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.02108692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"1","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021098438Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"0","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021183247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"2","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021448654Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[0]R flush_numbers[0..12]: [95, 101, 100, 100, 101, 101, 101, 100, 97, 96, 96, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021473641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[0]R generation[0..12]: [59, 63, 62, 62, 63, 63, 63, 62, 61, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021484449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021495318Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[1]R flush_numbers[0..12]: [95, 101, 100, 100, 101, 101, 101, 100, 97, 96, 96, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021505747Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[1]R generation[0..12]: [59, 63, 62, 62, 63, 63, 63, 62, 61, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021515955Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021526714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[2]R flush_numbers[0..12]: [95, 101, 100, 100, 101, 101, 101, 100, 97, 96, 96, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021537213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[2]R generation[0..12]: [59, 63, 62, 62, 63, 63, 63, 62, 61, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021547411Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.02156018Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021569498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021579147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021588476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021597885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"":"downstairs","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021607484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021616163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"0","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021625591Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"1","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.02163517Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"client":"2","":"downstairs","session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021644939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021654078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"1cfeba8f-ce01-47b7-b0cf-e623917c8f91 is now active with session: a37de63f-ac93-4097-ac36-ff75919e162d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021663946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021672685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"session_id":"a37de63f-ac93-4097-ac36-ff75919e162d"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021681754Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Activated sub_volume 1cfeba8f-ce01-47b7-b0cf-e623917c8f91","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021689963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.021698172Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.022001254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"task":"crutest"} 01/30 Write block 88 len 28672 data: 9 9 8 6 3 3 3 02/30 Read block 52 len 40960 03/30 Write block 57 len 16384 data: 7 8 8 7 04/30 Read block 397 len 20480 05/30 Read block 74 len 12288 06/30 Read block 193 len 24576 07/30 Read block 296 len 8192 08/30 Flush 09/30 Flush 10/30 Write block 123 len 40960 data: 5 5 6 6 7 7 9 8 11 11 11/30 Read block 82 len 40960 12/30 Write block 420 len 20480 data: 6 8 8 9 9 13/30 Read block 508 len 12288 14/30 Read block 311 len 16384 15/30 Read block 565 len 36864 16/30 Read block 326 len 28672 17/30 Read block 583 len 32768 18/30 Flush 19/30 Write block 395 len 20480 data: 6 4 4 3 2 20/30 Write block 74 len 12288 data: 8 9 7 21/30 Write block 419 len 36864 data: 6 7 9 9 10 10 10 8 7 22/30 Read block 551 len 32768 23/30 Read block 508 len 36864 24/30 Read block 469 len 16384 25/30 Write block 182 len 12288 data: 6 9 7 26/30 Read block 284 len 36864 27/30 Read block 384 len 16384 28/30 Write block 343 len 16384 data: 7 5 7 7 29/30 Read block 420 len 12288 30/30 Write block 440 len 40960 data: 7 5 4 5 5 6 7 8 7 7 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Read 10 Done Done Sent false 1002 Acked Write 4 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:41.031724331Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:41.031887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1420,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 63 63 63 101 101 101 F F F 2 040-059 64 64 62 102 102 100 F F F <--- 3 060-079 64 64 62 102 102 100 T T F <--- 4 080-099 64 64 63 102 102 101 F F F <--- 5 100-119 63 63 63 101 101 101 F F F 6 120-139 64 64 63 104 104 101 F F F <--- 7 140-159 62 62 62 100 100 100 F F F 8 160-179 61 61 61 97 97 97 F F F 9 180-199 60 60 60 96 96 96 T T F <--- 10 200-219 60 60 60 96 96 96 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 63 63 63 101 101 101 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 62 62 62 99 99 99 F F F 16 320-339 62 62 62 99 99 99 F F F 17 340-359 62 62 62 100 100 100 T T F <--- 18 360-379 58 58 58 93 93 93 F F F 19 380-399 58 58 58 94 94 94 T T F <--- 20 400-419 63 63 63 101 101 101 T T F <--- 21 420-439 64 64 60 104 104 96 T T F <--- 22 440-459 61 61 61 97 97 97 T T F <--- 23 460-479 61 61 61 97 97 97 F F F 24 480-499 59 59 59 95 95 95 F F F 25 500-519 61 61 61 97 97 97 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 58 58 58 94 94 94 F F F 29 580-599 60 60 60 96 96 96 F F F Max gen: 64, Max flush: 104 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 63 63 63 101 101 101 F F F 2 040-059 64 64 64 102 102 102 F F F 3 060-079 65 65 65 105 105 105 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 63 63 63 101 101 101 F F F 6 120-139 64 64 64 104 104 104 F F F 7 140-159 62 62 62 100 100 100 F F F 8 160-179 61 61 61 97 97 97 F F F 9 180-199 65 65 65 105 105 105 F F F 10 200-219 60 60 60 96 96 96 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 63 63 63 101 101 101 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 56 56 56 90 90 90 F F F 15 300-319 62 62 62 99 99 99 F F F 16 320-339 62 62 62 99 99 99 F F F 17 340-359 65 65 65 105 105 105 F F F 18 360-379 58 58 58 93 93 93 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 65 65 65 105 105 105 F F F 21 420-439 65 65 65 105 105 105 F F F 22 440-459 65 65 65 105 105 105 F F F 23 460-479 61 61 61 97 97 97 F F F 24 480-499 59 59 59 95 95 95 F F F 25 500-519 61 61 61 97 97 97 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 58 58 58 94 94 94 F F F 29 580-599 60 60 60 96 96 96 F F F Max gen: 65, Max flush: 105 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.281136294Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.288984248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289272502Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289297389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289306027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289317406Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Crucible 171d5b49-777d-48b6-8931-66bad8a8ce4a has session id: cd66b07e-2a0d-4586-92fa-0879016a71ab","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289739103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Upstairs opts: Upstairs UUID: 171d5b49-777d-48b6-8931-66bad8a8ce4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289767289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Crucible stats registered with UUID: 171d5b49-777d-48b6-8931-66bad8a8ce4a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289778178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:55.289787476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289937177Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289968333Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.289979332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290105606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290121494Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"171d5b49-777d-48b6-8931-66bad8a8ce4a active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290144761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.29015393Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"0","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290163659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"1","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290173277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"2","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290182696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290192565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290202234Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:36740)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290249928Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:52567)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290261596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:46256)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290361324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.29086193Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"2","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290882268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"0","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.290902225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"1","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291213716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[0]R flush_numbers[0..12]: [95, 101, 102, 105, 102, 101, 104, 100, 97, 105, 96, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291232103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[0]R generation[0..12]: [59, 63, 64, 65, 64, 63, 64, 62, 61, 65, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291245342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.2912564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[1]R flush_numbers[0..12]: [95, 101, 102, 105, 102, 101, 104, 100, 97, 105, 96, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291267279Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[1]R generation[0..12]: [59, 63, 64, 65, 64, 63, 64, 62, 61, 65, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291277647Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291288466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[2]R flush_numbers[0..12]: [95, 101, 102, 105, 102, 101, 104, 100, 97, 105, 96, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291298855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[2]R generation[0..12]: [59, 63, 64, 65, 64, 63, 64, 62, 61, 65, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291309173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291320002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291329351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.29133913Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291348658Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291358147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"":"downstairs","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291368076Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291376925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"0","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291388083Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"1","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291397892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"client":"2","":"downstairs","session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291407971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.29141724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"171d5b49-777d-48b6-8931-66bad8a8ce4a is now active with session: cd66b07e-2a0d-4586-92fa-0879016a71ab","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291427058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291435747Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"session_id":"cd66b07e-2a0d-4586-92fa-0879016a71ab"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291444746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Activated sub_volume 171d5b49-777d-48b6-8931-66bad8a8ce4a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291452815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291460884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.291745888Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"task":"crutest"} 01/30 Write block 283 len 20480 data: 7 4 5 6 7 02/30 Read block 28 len 20480 03/30 Read block 573 len 20480 04/30 Read block 590 len 36864 05/30 Write block 245 len 12288 data: 5 4 4 06/30 Read block 411 len 8192 07/30 Write block 514 len 12288 data: 4 5 5 08/30 Read block 514 len 32768 09/30 Read block 26 len 12288 10/30 Read block 309 len 32768 11/30 Write block 576 len 32768 data: 7 7 6 7 8 8 7 7 12/30 Read block 89 len 24576 13/30 Write block 507 len 24576 data: 5 7 5 5 5 6 14/30 Write block 499 len 4096 data: 2 15/30 Write block 465 len 12288 data: 6 5 4 16/30 Flush 17/30 Write block 416 len 16384 data: 6 5 6 7 18/30 Flush 19/30 Write block 357 len 8192 data: 5 5 20/30 Read block 583 len 16384 21/30 Read block 243 len 8192 22/30 Read block 47 len 36864 23/30 Write block 472 len 8192 data: 5 4 24/30 Read block 222 len 24576 25/30 Write block 416 len 16384 data: 7 6 7 8 26/30 Write block 306 len 4096 data: 7 27/30 Read block 204 len 24576 28/30 Read block 381 len 4096 29/30 Write block 198 len 40960 data: 5 6 6 6 6 7 7 8 8 8 30/30 Write block 134 len 20480 data: 8 7 6 5 7 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:52:55.299991772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:52:55.300182588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1429,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 63 63 63 101 101 101 F F F 2 040-059 64 64 64 102 102 102 F F F 3 060-079 65 65 65 105 105 105 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 63 63 63 101 101 101 F F F 6 120-139 64 64 64 104 104 104 T F T <--- 7 140-159 62 62 62 100 100 100 F F F 8 160-179 61 61 61 97 97 97 F F F 9 180-199 65 65 65 105 105 105 T F T <--- 10 200-219 60 60 60 96 96 96 T F T <--- 11 220-239 63 63 63 101 101 101 F F F 12 240-259 66 63 66 106 101 106 F F F <--- 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 56 66 106 90 106 F T F <--- 15 300-319 62 62 62 99 99 99 T F T <--- 16 320-339 62 62 62 99 99 99 F F F 17 340-359 65 65 65 105 105 105 T F T <--- 18 360-379 58 58 58 93 93 93 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 66 65 66 107 105 107 T F T <--- 21 420-439 65 65 65 105 105 105 F F F 22 440-459 65 65 65 105 105 105 F F F 23 460-479 66 61 66 106 97 106 T F T <--- 24 480-499 66 59 66 106 95 106 F F F <--- 25 500-519 66 61 66 106 97 106 F F F <--- 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 66 58 66 106 94 106 F F F <--- 29 580-599 66 60 66 106 96 106 F F F <--- Max gen: 66, Max flush: 107 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 63 63 63 101 101 101 F F F 2 040-059 64 64 64 102 102 102 F F F 3 060-079 65 65 65 105 105 105 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 63 63 63 101 101 101 F F F 6 120-139 67 67 67 108 108 108 F F F 7 140-159 62 62 62 100 100 100 F F F 8 160-179 61 61 61 97 97 97 F F F 9 180-199 67 67 67 108 108 108 F F F 10 200-219 67 67 67 108 108 108 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 66 66 66 106 106 106 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 66 66 106 106 106 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 62 62 62 99 99 99 F F F 17 340-359 67 67 67 108 108 108 F F F 18 360-379 58 58 58 93 93 93 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 65 65 65 105 105 105 F F F 22 440-459 65 65 65 105 105 105 F F F 23 460-479 67 67 67 108 108 108 F F F 24 480-499 66 66 66 106 106 106 F F F 25 500-519 66 66 66 106 106 106 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 66 66 66 106 106 106 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 67, Max flush: 108 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.555106112Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:09.562184684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.562251126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.569240439Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.569512675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.569539571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.56954802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.569562228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Crucible 76da33e7-1c1b-414d-984d-62466b484f93 has session id: c9a71766-b73f-4fb2-8944-5b4adb452dbf","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.569912034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Upstairs opts: Upstairs UUID: 76da33e7-1c1b-414d-984d-62466b484f93, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.56994043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Crucible stats registered with UUID: 76da33e7-1c1b-414d-984d-62466b484f93","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.569953759Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:09.569962708Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570082702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.57009961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570109739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570273038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570296455Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"76da33e7-1c1b-414d-984d-62466b484f93 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570306234Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570315093Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"0","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570324882Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"1","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.57033447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"2","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570343919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570355518Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570365557Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"ds_connection connected from Ok(127.0.0.1:39504)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570423639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"ds_connection connected from Ok(127.0.0.1:42576)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570525616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"ds_connection connected from Ok(127.0.0.1:39445)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.570541994Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571084695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"1","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.57112478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"2","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571158716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"0","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571468587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[0]R flush_numbers[0..12]: [95, 101, 102, 105, 102, 101, 108, 100, 97, 108, 108, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571503082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[0]R generation[0..12]: [59, 63, 64, 65, 64, 63, 67, 62, 61, 67, 67, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571514211Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571525259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[1]R flush_numbers[0..12]: [95, 101, 102, 105, 102, 101, 108, 100, 97, 108, 108, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571535698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[1]R generation[0..12]: [59, 63, 64, 65, 64, 63, 67, 62, 61, 67, 67, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571546047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571559245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[2]R flush_numbers[0..12]: [95, 101, 102, 105, 102, 101, 108, 100, 97, 108, 108, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571569804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[2]R generation[0..12]: [59, 63, 64, 65, 64, 63, 67, 62, 61, 67, 67, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571580032Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571590891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.57160033Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571609999Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571619248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571628706Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"":"downstairs","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571638575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571647384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"0","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571657213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"1","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571666911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"client":"2","":"downstairs","session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.57167699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571686089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"76da33e7-1c1b-414d-984d-62466b484f93 is now active with session: c9a71766-b73f-4fb2-8944-5b4adb452dbf","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571695998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571707726Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"session_id":"c9a71766-b73f-4fb2-8944-5b4adb452dbf"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571716805Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Activated sub_volume 76da33e7-1c1b-414d-984d-62466b484f93","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571724664Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.571732653Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.572020827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} 01/30 Write block 75 len 20480 data: 10 8 7 7 7 02/30 Write block 514 len 16384 data: 5 6 6 5 03/30 Read block 79 len 32768 04/30 Write block 248 len 8192 data: 7 6 05/30 Write block 193 len 20480 data: 9 7 6 6 4 06/30 Write block 58 len 40960 data: 9 9 8 6 6 6 5 5 5 5 07/30 Read block 220 len 32768 08/30 Write block 154 len 28672 data: 9 6 8 8 7 6 6 09/30 Read block 89 len 36864 10/30 Write block 365 len 12288 data: 6 6 7 11/30 Read block 403 len 36864 12/30 Write block 479 len 8192 data: 4 4 13/30 Flush 14/30 Write block 575 len 16384 data: 7 8 8 7 15/30 Flush 16/30 Write block 44 len 24576 data: 8 6 7 7 7 8 17/30 Read block 542 len 4096 18/30 Flush 19/30 Write block 160 len 24576 data: 7 7 7 9 10 7 20/30 Read block 78 len 32768 21/30 Write block 168 len 32768 data: 11 12 12 11 11 10 10 10 22/30 Flush 23/30 Write block 118 len 36864 data: 5 3 4 4 6 6 6 7 7 24/30 Write block 332 len 28672 data: 5 5 6 7 7 7 8 25/30 Read block 244 len 32768 26/30 Read block 109 len 8192 27/30 Read block 64 len 36864 28/30 Write block 27 len 8192 data: 4 4 29/30 Read block 231 len 40960 30/30 Write block 477 len 28672 data: 6 4 5 5 4 4 6 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 1 1 24 Done 8 29 29 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:09.580753099Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:09.580925007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1438,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 63 63 63 101 101 101 F T T <--- 2 040-059 64 68 68 102 111 111 T F F <--- 3 060-079 65 68 68 105 109 109 T F F <--- 4 080-099 64 64 64 102 102 102 F F F 5 100-119 63 63 63 101 101 101 F T T <--- 6 120-139 67 67 67 108 108 108 F T T <--- 7 140-159 62 68 68 100 109 109 T F F <--- 8 160-179 61 68 68 97 112 112 T F F <--- 9 180-199 67 68 68 108 109 109 T F F <--- 10 200-219 67 67 67 108 108 108 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 66 68 68 106 109 109 T F F <--- 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 66 66 106 106 106 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 62 62 62 99 99 99 F T T <--- 17 340-359 67 67 67 108 108 108 F F F 18 360-379 58 68 68 93 109 109 F F F <--- 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 65 65 65 105 105 105 F F F 22 440-459 65 65 65 105 105 105 F F F 23 460-479 67 68 68 108 109 109 F T T <--- 24 480-499 66 68 68 106 109 109 F T T <--- 25 500-519 66 68 68 106 109 109 T F F <--- 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 66 68 68 106 110 110 F F F <--- 29 580-599 66 66 66 106 106 106 F F F Max gen: 68, Max flush: 112 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 68 68 68 111 111 111 F F F 3 060-079 68 68 68 109 109 109 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 69 69 69 113 113 113 F F F 6 120-139 69 69 69 113 113 113 F F F 7 140-159 68 68 68 109 109 109 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 68 68 68 109 109 109 F F F 10 200-219 67 67 67 108 108 108 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 66 66 106 106 106 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 69 69 69 113 113 113 F F F 17 340-359 67 67 67 108 108 108 F F F 18 360-379 68 68 68 109 109 109 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 65 65 65 105 105 105 F F F 22 440-459 65 65 65 105 105 105 F F F 23 460-479 69 69 69 113 113 113 F F F 24 480-499 69 69 69 113 113 113 F F F 25 500-519 68 68 68 109 109 109 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 60 60 96 96 96 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 69, Max flush: 113 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.841756551Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:23.848736396Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.84878593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.855420808Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.855672446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.855701143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.855709622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.85572095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Crucible 3ffdfab6-17f9-4e7c-9666-f8a3aabf055a has session id: b401194b-2797-46db-abfd-f18ce1571c8a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856080884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Upstairs opts: Upstairs UUID: 3ffdfab6-17f9-4e7c-9666-f8a3aabf055a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.85611827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Crucible stats registered with UUID: 3ffdfab6-17f9-4e7c-9666-f8a3aabf055a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856133248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:23.856142077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856294307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856310635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856320594Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856446048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856470005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"3ffdfab6-17f9-4e7c-9666-f8a3aabf055a active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856481834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856490452Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"0","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856500111Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"1","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.85650965Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"2","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856519229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856529148Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856539096Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61715)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856626695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55447)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856652152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51200)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.856734292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857329086Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"1","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857346674Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"2","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857462389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"0","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857732985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[0]R flush_numbers[0..12]: [95, 113, 111, 109, 102, 113, 113, 109, 112, 109, 108, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857764261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[0]R generation[0..12]: [59, 69, 68, 68, 64, 69, 69, 68, 68, 68, 67, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.8577751Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857786248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[1]R flush_numbers[0..12]: [95, 113, 111, 109, 102, 113, 113, 109, 112, 109, 108, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857796837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[1]R generation[0..12]: [59, 69, 68, 68, 64, 69, 69, 68, 68, 68, 67, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857807086Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857817884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[2]R flush_numbers[0..12]: [95, 113, 111, 109, 102, 113, 113, 109, 112, 109, 108, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857828213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[2]R generation[0..12]: [59, 69, 68, 68, 64, 69, 69, 68, 68, 68, 67, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857838332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.85784918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857861119Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857870797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857880126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857889425Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"":"downstairs","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857899304Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857908023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"0","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857917971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"1","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.85792768Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"client":"2","":"downstairs","session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857937639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857946738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"3ffdfab6-17f9-4e7c-9666-f8a3aabf055a is now active with session: b401194b-2797-46db-abfd-f18ce1571c8a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857956577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857965235Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"session_id":"b401194b-2797-46db-abfd-f18ce1571c8a"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857974094Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Activated sub_volume 3ffdfab6-17f9-4e7c-9666-f8a3aabf055a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857982123Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.857990312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.858272456Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} 01/30 Write block 453 len 28672 data: 9 10 7 6 6 6 7 02/30 Read block 15 len 32768 03/30 Read block 22 len 40960 04/30 Read block 219 len 8192 05/30 Read block 189 len 12288 06/30 Write block 516 len 8192 data: 7 6 07/30 Write block 548 len 12288 data: 4 4 5 08/30 Read block 87 len 36864 09/30 Write block 347 len 12288 data: 5 3 3 10/30 Write block 207 len 28672 data: 9 9 7 6 6 7 7 11/30 Read block 134 len 40960 12/30 Read block 540 len 20480 13/30 Read block 15 len 24576 14/30 Read block 542 len 8192 15/30 Read block 351 len 40960 16/30 Read block 148 len 36864 17/30 Flush 18/30 Write block 120 len 4096 data: 5 19/30 Write block 110 len 40960 data: 8 7 6 6 5 5 4 4 6 4 20/30 Write block 123 len 20480 data: 7 7 8 8 8 21/30 Read block 72 len 40960 22/30 Read block 481 len 28672 23/30 Read block 119 len 16384 24/30 Read block 366 len 24576 25/30 Flush 26/30 Write block 156 len 12288 data: 9 9 8 27/30 Read block 313 len 16384 28/30 Write block 438 len 40960 data: 6 7 8 6 5 6 6 7 8 9 29/30 Read block 151 len 32768 30/30 Write block 326 len 20480 data: 4 6 6 6 7 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 5 29 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:23.867184126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:23.867377222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1447,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 68 68 68 111 111 111 F F F 3 060-079 68 68 68 109 109 109 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 69 70 70 113 115 115 F F F <--- 6 120-139 69 70 70 113 115 115 F F F <--- 7 140-159 68 68 68 109 109 109 F T T <--- 8 160-179 68 68 68 112 112 112 F F F 9 180-199 68 68 68 109 109 109 F F F 10 200-219 67 70 70 108 114 114 F F F <--- 11 220-239 63 63 63 101 101 101 F F F 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 66 66 106 106 106 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 69 69 69 113 113 113 F T T <--- 17 340-359 67 70 70 108 114 114 F F F <--- 18 360-379 68 68 68 109 109 109 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 65 65 65 105 105 105 F T T <--- 22 440-459 65 70 70 105 114 114 T T T <--- 23 460-479 69 69 69 113 113 113 F F F 24 480-499 69 69 69 113 113 113 F F F 25 500-519 68 70 70 109 114 114 F F F <--- 26 520-539 61 61 61 97 97 97 F F F 27 540-559 60 70 70 96 114 114 F F F <--- 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 70, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 68 68 68 111 111 111 F F F 3 060-079 68 68 68 109 109 109 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 70 70 70 115 115 115 F F F 6 120-139 70 70 70 115 115 115 F F F 7 140-159 71 71 71 116 116 116 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 68 68 68 109 109 109 F F F 10 200-219 70 70 70 114 114 114 F F F 11 220-239 63 63 63 101 101 101 F F F 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 66 66 106 106 106 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 71 71 71 116 116 116 F F F 17 340-359 70 70 70 114 114 114 F F F 18 360-379 68 68 68 109 109 109 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 71 71 71 116 116 116 F F F 22 440-459 71 71 71 116 116 116 F F F 23 460-479 69 69 69 113 113 113 F F F 24 480-499 69 69 69 113 113 113 F F F 25 500-519 70 70 70 114 114 114 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 71, Max flush: 116 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.116256332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:38.123346433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.123401006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130081799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130339996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130364273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130372542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130383891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Crucible 72c64452-0668-4e29-b179-b448d3af6e56 has session id: 7ad65e97-140a-490d-9f2c-aa28a90dacec","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130731166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Upstairs opts: Upstairs UUID: 72c64452-0668-4e29-b179-b448d3af6e56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130746245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Crucible stats registered with UUID: 72c64452-0668-4e29-b179-b448d3af6e56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130758653Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:38.130767582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130916343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130949259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.130960247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.1310966Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131112398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"72c64452-0668-4e29-b179-b448d3af6e56 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131135795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131145124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"0","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131154633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"1","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131164162Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"2","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.13117376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131183859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131193668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"ds_connection connected from Ok(127.0.0.1:36904)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131269698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"ds_connection connected from Ok(127.0.0.1:35831)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131288046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"ds_connection connected from Ok(127.0.0.1:63217)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131359387Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.131994996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"1","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132031662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"2","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"0","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132378398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[0]R flush_numbers[0..12]: [95, 113, 111, 109, 102, 115, 115, 116, 112, 109, 114, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132406134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[0]R generation[0..12]: [59, 69, 68, 68, 64, 70, 70, 71, 68, 68, 70, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132417283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132428281Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[1]R flush_numbers[0..12]: [95, 113, 111, 109, 102, 115, 115, 116, 112, 109, 114, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.13243889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[1]R generation[0..12]: [59, 69, 68, 68, 64, 70, 70, 71, 68, 68, 70, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132449399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132460307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[2]R flush_numbers[0..12]: [95, 113, 111, 109, 102, 115, 115, 116, 112, 109, 114, 101]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132470956Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[2]R generation[0..12]: [59, 69, 68, 68, 64, 70, 70, 71, 68, 68, 70, 63]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132481205Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132492173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132501542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132511221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.13252049Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132529828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"":"downstairs","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132539577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132548386Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"0","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132559255Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"1","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132568923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"client":"2","":"downstairs","session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132578692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132587921Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"72c64452-0668-4e29-b179-b448d3af6e56 is now active with session: 7ad65e97-140a-490d-9f2c-aa28a90dacec","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.13259777Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132606509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"session_id":"7ad65e97-140a-490d-9f2c-aa28a90dacec"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132615517Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Activated sub_volume 72c64452-0668-4e29-b179-b448d3af6e56","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132623536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132631595Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.132918669Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} 01/30 Write block 46 len 8192 data: 8 8 02/30 Read block 80 len 16384 03/30 Write block 283 len 16384 data: 8 5 6 7 04/30 Read block 318 len 8192 05/30 Read block 594 len 24576 06/30 Read block 264 len 16384 07/30 Write block 347 len 40960 data: 6 4 4 3 3 6 5 5 5 5 08/30 Read block 333 len 36864 09/30 Flush 10/30 Write block 207 len 12288 data: 10 10 8 11/30 Write block 49 len 36864 data: 9 8 9 7 7 7 7 6 8 12/30 Read block 52 len 32768 13/30 Read block 444 len 24576 14/30 Read block 185 len 4096 15/30 Read block 479 len 36864 16/30 Write block 131 len 24576 data: 12 12 9 9 8 7 17/30 Read block 414 len 36864 18/30 Read block 196 len 32768 19/30 Write block 137 len 28672 data: 6 8 9 6 10 10 8 20/30 Write block 484 len 20480 data: 6 7 5 5 3 21/30 Read block 212 len 8192 22/30 Write block 357 len 28672 data: 6 6 6 5 5 6 7 23/30 Write block 214 len 40960 data: 7 6 6 6 5 5 7 8 7 6 24/30 Write block 112 len 40960 data: 7 7 6 6 5 5 7 5 6 5 25/30 Read block 473 len 20480 26/30 Flush 27/30 Write block 460 len 16384 data: 5 4 6 8 28/30 Read block 287 len 40960 29/30 Read block 517 len 28672 30/30 Read block 89 len 32768 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Read 4 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:38.142309138Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:38.142475027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1456,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 68 72 72 111 118 118 F F F <--- 3 060-079 68 68 68 109 109 109 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 70 72 72 115 118 118 F F F <--- 6 120-139 70 72 72 115 118 118 F F F <--- 7 140-159 71 72 72 116 118 118 F F F <--- 8 160-179 68 68 68 112 112 112 F F F 9 180-199 68 68 68 109 109 109 F F F 10 200-219 70 72 72 114 118 118 F F F <--- 11 220-239 63 72 72 101 118 118 F F F <--- 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 66 72 72 106 117 117 F F F <--- 15 300-319 67 67 67 108 108 108 F F F 16 320-339 71 71 71 116 116 116 F F F 17 340-359 70 72 72 114 118 118 F F F <--- 18 360-379 68 72 72 109 118 118 F F F <--- 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 71 71 71 116 116 116 F F F 22 440-459 71 71 71 116 116 116 F F F 23 460-479 69 69 69 113 113 113 F T T <--- 24 480-499 69 72 72 113 118 118 F F F <--- 25 500-519 70 70 70 114 114 114 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 72, Max flush: 118 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 72 72 72 118 118 118 F F F 3 060-079 68 68 68 109 109 109 F F F 4 080-099 64 64 64 102 102 102 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 72 72 72 118 118 118 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 68 68 68 109 109 109 F F F 10 200-219 72 72 72 118 118 118 F F F 11 220-239 72 72 72 118 118 118 F F F 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 72 72 72 117 117 117 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 71 71 71 116 116 116 F F F 17 340-359 72 72 72 118 118 118 F F F 18 360-379 72 72 72 118 118 118 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 F F F 21 420-439 71 71 71 116 116 116 F F F 22 440-459 71 71 71 116 116 116 F F F 23 460-479 73 73 73 119 119 119 F F F 24 480-499 72 72 72 118 118 118 F F F 25 500-519 70 70 70 114 114 114 F F F 26 520-539 61 61 61 97 97 97 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 73, Max flush: 119 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.394946142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.402918001Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403183227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403208334Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403216843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403228321Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Crucible e79c2842-4fa9-4ec9-ba39-d310dacb5097 has session id: d0fece2a-7c6d-41a7-b91c-2574f4c8e407","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403587496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Upstairs opts: Upstairs UUID: e79c2842-4fa9-4ec9-ba39-d310dacb5097, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403615832Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Crucible stats registered with UUID: e79c2842-4fa9-4ec9-ba39-d310dacb5097","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403626641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:52.40363575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.40378868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403818037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403828965Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403942891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403958309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"e79c2842-4fa9-4ec9-ba39-d310dacb5097 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403973967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403982646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"0","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.403992195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"1","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404001593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"2","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404011112Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404021131Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404033019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"ds_connection connected from Ok(127.0.0.1:51167)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404076544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"ds_connection connected from Ok(127.0.0.1:65375)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404091142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"ds_connection connected from Ok(127.0.0.1:33131)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404101231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404726971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"1","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404769886Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"0","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.404794763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"2","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405088675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[0]R flush_numbers[0..12]: [95, 113, 118, 109, 102, 118, 118, 118, 112, 109, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405110373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[0]R generation[0..12]: [59, 69, 72, 68, 64, 72, 72, 72, 68, 68, 72, 72]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405121671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.40513277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[1]R flush_numbers[0..12]: [95, 113, 118, 109, 102, 118, 118, 118, 112, 109, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405143488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[1]R generation[0..12]: [59, 69, 72, 68, 64, 72, 72, 72, 68, 68, 72, 72]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405153847Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405164736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[2]R flush_numbers[0..12]: [95, 113, 118, 109, 102, 118, 118, 118, 112, 109, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405177814Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[2]R generation[0..12]: [59, 69, 72, 68, 64, 72, 72, 72, 68, 68, 72, 72]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405188153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405198921Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.40520823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405217949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405227108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405236407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"":"downstairs","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405246175Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405254884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"0","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405264613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"1","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405274152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"client":"2","":"downstairs","session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405283911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.40529305Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"e79c2842-4fa9-4ec9-ba39-d310dacb5097 is now active with session: d0fece2a-7c6d-41a7-b91c-2574f4c8e407","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405302978Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405311717Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"session_id":"d0fece2a-7c6d-41a7-b91c-2574f4c8e407"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405320756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Activated sub_volume e79c2842-4fa9-4ec9-ba39-d310dacb5097","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405330155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405338304Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.405620878Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"task":"crutest"} 01/30 Write block 357 len 40960 data: 7 7 7 6 6 7 8 8 7 7 02/30 Read block 570 len 8192 03/30 Read block 331 len 4096 04/30 Read block 578 len 24576 05/30 Read block 39 len 12288 06/30 Write block 192 len 20480 data: 8 10 8 7 7 07/30 Write block 218 len 32768 data: 6 6 8 9 8 7 7 6 08/30 Read block 207 len 20480 09/30 Read block 326 len 12288 10/30 Write block 351 len 12288 data: 4 7 6 11/30 Write block 74 len 36864 data: 9 11 9 8 8 8 5 3 4 12/30 Write block 283 len 28672 data: 9 6 7 8 8 5 6 13/30 Read block 201 len 24576 14/30 Read block 212 len 4096 15/30 Read block 260 len 28672 16/30 Read block 62 len 32768 17/30 Write block 417 len 28672 data: 7 8 9 8 10 10 11 18/30 Write block 520 len 24576 data: 5 5 3 4 4 6 19/30 Write block 348 len 12288 data: 5 5 4 20/30 Read block 198 len 28672 21/30 Read block 386 len 32768 22/30 Write block 514 len 36864 data: 6 7 8 7 4 5 6 6 4 23/30 Read block 19 len 16384 24/30 Read block 388 len 32768 25/30 Read block 457 len 16384 26/30 Read block 123 len 40960 27/30 Write block 404 len 20480 data: 7 6 6 6 5 28/30 Write block 524 len 40960 data: 5 7 6 6 6 6 8 8 8 9 29/30 Write block 291 len 32768 data: 5 6 8 8 8 8 6 5 30/30 Write block 297 len 20480 data: 7 6 5 4 4 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Read 2 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 30 36 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:53:52.413985597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:53:52.414161605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1465,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 72 72 72 118 118 118 F F F 3 060-079 68 68 68 109 109 109 T T F <--- 4 080-099 64 64 64 102 102 102 T T F <--- 5 100-119 72 72 72 118 118 118 F F F 6 120-139 72 72 72 118 118 118 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 68 68 68 109 109 109 T T F <--- 10 200-219 72 72 72 118 118 118 T T F <--- 11 220-239 72 72 72 118 118 118 T T F <--- 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 72 72 72 117 117 117 T T F <--- 15 300-319 67 67 67 108 108 108 T T F <--- 16 320-339 71 71 71 116 116 116 F F F 17 340-359 72 72 72 118 118 118 T T F <--- 18 360-379 72 72 72 118 118 118 T T F <--- 19 380-399 65 65 65 105 105 105 F F F 20 400-419 67 67 67 108 108 108 T T F <--- 21 420-439 71 71 71 116 116 116 T T F <--- 22 440-459 71 71 71 116 116 116 F F F 23 460-479 73 73 73 119 119 119 F F F 24 480-499 72 72 72 118 118 118 F F F 25 500-519 70 70 70 114 114 114 T T F <--- 26 520-539 61 61 61 97 97 97 T T F <--- 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 73, Max flush: 119 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 72 72 72 118 118 118 F F F 3 060-079 74 74 74 120 120 120 F F F 4 080-099 74 74 74 120 120 120 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 72 72 72 118 118 118 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 74 74 74 120 120 120 F F F 10 200-219 74 74 74 120 120 120 F F F 11 220-239 74 74 74 120 120 120 F F F 12 240-259 68 68 68 109 109 109 F F F 13 260-279 62 62 62 99 99 99 F F F 14 280-299 74 74 74 120 120 120 F F F 15 300-319 74 74 74 120 120 120 F F F 16 320-339 71 71 71 116 116 116 F F F 17 340-359 74 74 74 120 120 120 F F F 18 360-379 74 74 74 120 120 120 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 74 74 74 120 120 120 F F F 21 420-439 74 74 74 120 120 120 F F F 22 440-459 71 71 71 116 116 116 F F F 23 460-479 73 73 73 119 119 119 F F F 24 480-499 72 72 72 118 118 118 F F F 25 500-519 74 74 74 120 120 120 F F F 26 520-539 74 74 74 120 120 120 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 74, Max flush: 120 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.674804673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.682766343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683051077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683076904Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683085293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683096702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Crucible d7b3fa48-7d1a-4604-b73f-a87ea7445f9f has session id: 447a9fb9-1ddb-4141-a130-6bc78fbba904","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683416981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Upstairs opts: Upstairs UUID: d7b3fa48-7d1a-4604-b73f-a87ea7445f9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683485192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Crucible stats registered with UUID: d7b3fa48-7d1a-4604-b73f-a87ea7445f9f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683510099Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:06.683520038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.6836576Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683690486Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683701595Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683807601Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683824109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"d7b3fa48-7d1a-4604-b73f-a87ea7445f9f active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683838128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683846926Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"0","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683856485Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"1","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683866104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"2","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683875673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.68389505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683906859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"ds_connection connected from Ok(127.0.0.1:54541)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683928626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"ds_connection connected from Ok(127.0.0.1:60921)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683938835Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"ds_connection connected from Ok(127.0.0.1:36231)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.683996267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.684672702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"1","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.684696529Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"0","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.684710407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"2","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.684992981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[0]R flush_numbers[0..12]: [95, 113, 118, 120, 120, 118, 118, 118, 112, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685016258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[0]R generation[0..12]: [59, 69, 72, 74, 74, 72, 72, 72, 68, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685026807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685037705Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[1]R flush_numbers[0..12]: [95, 113, 118, 120, 120, 118, 118, 118, 112, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685048294Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[1]R generation[0..12]: [59, 69, 72, 74, 74, 72, 72, 72, 68, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685058413Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685069151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[2]R flush_numbers[0..12]: [95, 113, 118, 120, 120, 118, 118, 118, 112, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.68507966Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[2]R generation[0..12]: [59, 69, 72, 74, 74, 72, 72, 72, 68, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685090329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685101097Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Max found gen is 75","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685110366Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Generation requested: 76 >= found:75","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685119895Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685129154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685139952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"":"downstairs","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685149461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.68515806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"0","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685167659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"1","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685177478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"client":"2","":"downstairs","session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685187426Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685196545Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"d7b3fa48-7d1a-4604-b73f-a87ea7445f9f is now active with session: 447a9fb9-1ddb-4141-a130-6bc78fbba904","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685206344Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685215053Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"session_id":"447a9fb9-1ddb-4141-a130-6bc78fbba904"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685224052Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Activated sub_volume d7b3fa48-7d1a-4604-b73f-a87ea7445f9f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685232041Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.68524007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.685513075Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"task":"crutest"} 01/30 Flush 02/30 Write block 76 len 24576 data: 10 9 9 9 6 4 03/30 Read block 355 len 32768 04/30 Flush 05/30 Write block 269 len 16384 data: 5 5 4 4 06/30 Flush 07/30 Write block 506 len 28672 data: 5 6 8 6 6 6 7 08/30 Write block 318 len 24576 data: 4 4 4 3 4 4 09/30 Write block 240 len 40960 data: 6 4 5 4 4 6 5 5 8 7 10/30 Write block 419 len 28672 data: 10 9 11 11 12 11 11 11/30 Flush 12/30 Write block 108 len 16384 data: 10 10 9 8 13/30 Flush 14/30 Write block 340 len 40960 data: 6 7 6 8 6 8 8 7 6 6 15/30 Write block 538 len 4096 data: 10 16/30 Write block 312 len 36864 data: 4 3 4 4 3 3 5 5 5 17/30 Read block 119 len 32768 18/30 Write block 153 len 12288 data: 10 10 7 19/30 Read block 276 len 36864 20/30 Read block 210 len 36864 21/30 Read block 74 len 28672 22/30 Read block 433 len 36864 23/30 Read block 150 len 8192 24/30 Write block 267 len 12288 data: 4 4 6 25/30 Flush 26/30 Write block 345 len 8192 data: 9 9 27/30 Write block 463 len 12288 data: 9 6 7 28/30 Read block 563 len 24576 29/30 Write block 247 len 12288 data: 6 9 8 30/30 Write block 402 len 12288 data: 6 7 8 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:06.694112944Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:06.694263255Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1474,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 72 72 72 118 118 118 F F F 3 060-079 76 76 74 122 122 120 F F F <--- 4 080-099 76 76 74 122 122 120 F F F <--- 5 100-119 76 76 72 125 125 118 F F F <--- 6 120-139 72 72 72 118 118 118 F F F 7 140-159 76 76 72 126 126 118 F F F <--- 8 160-179 68 68 68 112 112 112 F F F 9 180-199 74 74 74 120 120 120 F F F 10 200-219 74 74 74 120 120 120 F F F 11 220-239 74 74 74 120 120 120 F F F 12 240-259 76 76 68 124 124 109 T T F <--- 13 260-279 76 76 62 126 126 99 F F F <--- 14 280-299 74 74 74 120 120 120 F F F 15 300-319 76 76 74 126 126 120 F F F <--- 16 320-339 76 76 71 126 126 116 F F F <--- 17 340-359 76 76 74 126 126 120 T T F <--- 18 360-379 74 74 74 120 120 120 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 76 76 74 124 124 120 T T F <--- 21 420-439 76 76 74 124 124 120 F F F <--- 22 440-459 71 71 71 116 116 116 F F F 23 460-479 73 73 73 119 119 119 T T F <--- 24 480-499 72 72 72 118 118 118 F F F 25 500-519 76 76 74 124 124 120 F F F <--- 26 520-539 76 76 74 126 126 120 F F F <--- 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 76, Max flush: 126 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 F F F 2 040-059 72 72 72 118 118 118 F F F 3 060-079 76 76 76 122 122 122 F F F 4 080-099 76 76 76 122 122 122 F F F 5 100-119 76 76 76 125 125 125 F F F 6 120-139 72 72 72 118 118 118 F F F 7 140-159 76 76 76 126 126 126 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 74 74 74 120 120 120 F F F 10 200-219 74 74 74 120 120 120 F F F 11 220-239 74 74 74 120 120 120 F F F 12 240-259 77 77 77 127 127 127 F F F 13 260-279 76 76 76 126 126 126 F F F 14 280-299 74 74 74 120 120 120 F F F 15 300-319 76 76 76 126 126 126 F F F 16 320-339 76 76 76 126 126 126 F F F 17 340-359 77 77 77 127 127 127 F F F 18 360-379 74 74 74 120 120 120 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 77 77 77 127 127 127 F F F 21 420-439 76 76 76 124 124 124 F F F 22 440-459 71 71 71 116 116 116 F F F 23 460-479 77 77 77 127 127 127 F F F 24 480-499 72 72 72 118 118 118 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 76 76 76 126 126 126 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 77, Max flush: 127 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.952573819Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.960710817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961011319Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961033056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961043935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961055303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Crucible c0a3f835-58c0-4198-ad24-c6f938a99512 has session id: a54a07a9-d033-4462-92e3-79e900867411","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961423117Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Upstairs opts: Upstairs UUID: c0a3f835-58c0-4198-ad24-c6f938a99512, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961449103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Crucible stats registered with UUID: c0a3f835-58c0-4198-ad24-c6f938a99512","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961458812Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:20.961467621Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961606303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961622131Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.96163189Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.96179322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961816587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"c0a3f835-58c0-4198-ad24-c6f938a99512 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961832695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961841774Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"0","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961851542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"1","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961861041Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"2","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.96187229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961882149Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961891977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"ds_connection connected from Ok(127.0.0.1:36743)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.961981546Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"ds_connection connected from Ok(127.0.0.1:57097)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.962075714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"ds_connection connected from Ok(127.0.0.1:48197)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.962092602Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.962675468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"1","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.962691966Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"0","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.962702764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"2","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963015775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[0]R flush_numbers[0..12]: [95, 113, 118, 122, 122, 125, 118, 126, 112, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.96305216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[0]R generation[0..12]: [59, 69, 72, 76, 76, 76, 72, 76, 68, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963063299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963074447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[1]R flush_numbers[0..12]: [95, 113, 118, 122, 122, 125, 118, 126, 112, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963085046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[1]R generation[0..12]: [59, 69, 72, 76, 76, 76, 72, 76, 68, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963097324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963108253Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[2]R flush_numbers[0..12]: [95, 113, 118, 122, 122, 125, 118, 126, 112, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963118712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[2]R generation[0..12]: [59, 69, 72, 76, 76, 76, 72, 76, 68, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.96312911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963139829Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963149258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963159047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963168425Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963177834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"":"downstairs","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963187563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963196292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"0","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963205961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"1","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963215469Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"client":"2","":"downstairs","session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963225198Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963234357Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"c0a3f835-58c0-4198-ad24-c6f938a99512 is now active with session: a54a07a9-d033-4462-92e3-79e900867411","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963244206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963254354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"session_id":"a54a07a9-d033-4462-92e3-79e900867411"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963263293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Activated sub_volume c0a3f835-58c0-4198-ad24-c6f938a99512","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963271222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963279301Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.963577514Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"task":"crutest"} 01/30 Flush 02/30 Write block 179 len 12288 data: 5 7 6 03/30 Write block 71 len 40960 data: 9 8 7 10 12 11 10 10 10 7 04/30 Read block 405 len 12288 05/30 Flush 06/30 Write block 129 len 16384 data: 10 9 13 13 07/30 Write block 213 len 16384 data: 8 8 7 7 08/30 Write block 56 len 24576 data: 7 9 10 10 9 7 09/30 Write block 75 len 36864 data: 13 12 11 11 11 8 5 5 4 10/30 Read block 17 len 40960 11/30 Read block 474 len 24576 12/30 Read block 551 len 12288 13/30 Write block 513 len 28672 data: 5 7 8 9 8 5 6 14/30 Write block 456 len 32768 data: 7 7 7 8 6 5 7 10 15/30 Read block 453 len 16384 16/30 Read block 297 len 12288 17/30 Read block 46 len 20480 18/30 Read block 141 len 8192 19/30 Write block 132 len 24576 data: 14 10 10 9 8 7 20/30 Write block 333 len 20480 data: 6 7 8 8 8 21/30 Write block 493 len 32768 data: 4 2 2 2 2 2 3 3 22/30 Read block 418 len 16384 23/30 Write block 27 len 20480 data: 5 5 5 5 5 24/30 Write block 49 len 4096 data: 10 25/30 Read block 418 len 28672 26/30 Read block 226 len 24576 27/30 Read block 96 len 20480 28/30 Write block 57 len 4096 data: 10 29/30 Write block 268 len 4096 data: 5 30/30 Read block 502 len 28672 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1000)) Some(JobId(1004)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:20.971312613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:20.97149246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1483,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 69 69 69 113 113 113 T F T <--- 2 040-059 72 72 72 118 118 118 T F T <--- 3 060-079 78 76 78 129 122 129 T F T <--- 4 080-099 78 76 78 129 122 129 T F T <--- 5 100-119 76 76 76 125 125 125 F F F 6 120-139 72 72 72 118 118 118 T F T <--- 7 140-159 76 76 76 126 126 126 F F F 8 160-179 78 68 78 129 112 129 F F F <--- 9 180-199 78 74 78 129 120 129 F F F <--- 10 200-219 74 74 74 120 120 120 T F T <--- 11 220-239 74 74 74 120 120 120 F F F 12 240-259 77 77 77 127 127 127 F F F 13 260-279 76 76 76 126 126 126 T F T <--- 14 280-299 74 74 74 120 120 120 F F F 15 300-319 76 76 76 126 126 126 F F F 16 320-339 76 76 76 126 126 126 T F T <--- 17 340-359 77 77 77 127 127 127 F F F 18 360-379 74 74 74 120 120 120 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 77 77 77 127 127 127 F F F 21 420-439 76 76 76 124 124 124 F F F 22 440-459 71 71 71 116 116 116 T F T <--- 23 460-479 77 77 77 127 127 127 T F T <--- 24 480-499 72 72 72 118 118 118 T F T <--- 25 500-519 76 76 76 124 124 124 T F T <--- 26 520-539 76 76 76 126 126 126 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 78, Max flush: 129 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 79 79 79 130 130 130 F F F 2 040-059 79 79 79 130 130 130 F F F 3 060-079 79 79 79 130 130 130 F F F 4 080-099 79 79 79 130 130 130 F F F 5 100-119 76 76 76 125 125 125 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 76 76 76 126 126 126 F F F 8 160-179 78 78 78 129 129 129 F F F 9 180-199 78 78 78 129 129 129 F F F 10 200-219 79 79 79 130 130 130 F F F 11 220-239 74 74 74 120 120 120 F F F 12 240-259 77 77 77 127 127 127 F F F 13 260-279 79 79 79 130 130 130 F F F 14 280-299 74 74 74 120 120 120 F F F 15 300-319 76 76 76 126 126 126 F F F 16 320-339 79 79 79 130 130 130 F F F 17 340-359 77 77 77 127 127 127 F F F 18 360-379 74 74 74 120 120 120 F F F 19 380-399 65 65 65 105 105 105 F F F 20 400-419 77 77 77 127 127 127 F F F 21 420-439 76 76 76 124 124 124 F F F 22 440-459 79 79 79 130 130 130 F F F 23 460-479 79 79 79 130 130 130 F F F 24 480-499 79 79 79 130 130 130 F F F 25 500-519 79 79 79 130 130 130 F F F 26 520-539 76 76 76 126 126 126 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 79, Max flush: 130 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.228023329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:35.235071565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.235125249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.241861574Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242119322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242143698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242152107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242163536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Crucible bd5df9c5-5680-48a3-a2b5-438709ad2810 has session id: bf4ebe5f-2ec8-4547-b281-907ce5577cca","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242514221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Upstairs opts: Upstairs UUID: bd5df9c5-5680-48a3-a2b5-438709ad2810, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242533509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Crucible stats registered with UUID: bd5df9c5-5680-48a3-a2b5-438709ad2810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242548097Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:35.242557076Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242703068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242735373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242746252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242882165Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242898793Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"bd5df9c5-5680-48a3-a2b5-438709ad2810 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242914481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.24292314Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"0","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242932908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"1","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242942477Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"2","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242952056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242961975Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.242971823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"ds_connection connected from Ok(127.0.0.1:42573)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.243117745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"ds_connection connected from Ok(127.0.0.1:33099)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.243211973Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"ds_connection connected from Ok(127.0.0.1:41578)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.243226431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.243753244Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"1","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.243765883Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"0","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.243776691Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"2","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244122038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[0]R flush_numbers[0..12]: [95, 130, 130, 130, 130, 125, 130, 126, 129, 129, 130, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244149084Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[0]R generation[0..12]: [59, 79, 79, 79, 79, 76, 79, 76, 78, 78, 79, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244160013Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244170931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[1]R flush_numbers[0..12]: [95, 130, 130, 130, 130, 125, 130, 126, 129, 129, 130, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.24418144Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[1]R generation[0..12]: [59, 79, 79, 79, 79, 76, 79, 76, 78, 78, 79, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244191719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244202587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[2]R flush_numbers[0..12]: [95, 130, 130, 130, 130, 125, 130, 126, 129, 129, 130, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244213176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[2]R generation[0..12]: [59, 79, 79, 79, 79, 76, 79, 76, 78, 78, 79, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244223415Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244234383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244245752Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244255461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244264739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244274158Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"":"downstairs","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244283697Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244292216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"0","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244301685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"1","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244311284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"client":"2","":"downstairs","session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244320922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244329941Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"bd5df9c5-5680-48a3-a2b5-438709ad2810 is now active with session: bf4ebe5f-2ec8-4547-b281-907ce5577cca","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.24433974Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244348329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"session_id":"bf4ebe5f-2ec8-4547-b281-907ce5577cca"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244357368Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Activated sub_volume bd5df9c5-5680-48a3-a2b5-438709ad2810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244365257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244373186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.244676077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} 01/30 Write block 420 len 8192 data: 10 12 02/30 Flush 03/30 Write block 384 len 40960 data: 4 3 3 4 4 6 7 7 9 9 04/30 Write block 431 len 24576 data: 4 4 4 5 5 5 05/30 Read block 510 len 32768 06/30 Read block 357 len 4096 07/30 Write block 275 len 36864 data: 6 7 7 7 7 8 8 8 10 08/30 Write block 96 len 8192 data: 5 5 09/30 Write block 515 len 36864 data: 9 10 9 6 7 7 7 5 5 10/30 Read block 210 len 12288 11/30 Read block 411 len 12288 12/30 Read block 258 len 24576 13/30 Read block 42 len 28672 14/30 Read block 380 len 20480 15/30 Read block 532 len 12288 16/30 Read block 51 len 16384 17/30 Flush 18/30 Write block 179 len 16384 data: 6 8 7 7 19/30 Read block 192 len 20480 20/30 Flush 21/30 Write block 308 len 40960 data: 8 6 6 6 5 4 5 5 4 4 22/30 Read block 90 len 28672 23/30 Write block 48 len 36864 data: 8 11 9 10 8 8 8 8 8 24/30 Flush 25/30 Write block 183 len 4096 data: 10 26/30 Read block 597 len 12288 27/30 Read block 381 len 36864 28/30 Read block 213 len 16384 29/30 Write block 56 len 40960 data: 9 11 11 11 10 8 7 7 6 6 30/30 Read block 188 len 40960 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:35.254269671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:35.254439929Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1492,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 79 79 79 130 130 130 F F F 2 040-059 79 80 80 130 134 134 F T T <--- 3 060-079 79 79 79 130 130 130 F T T <--- 4 080-099 79 80 80 130 132 132 F F F <--- 5 100-119 76 76 76 125 125 125 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 76 76 76 126 126 126 F F F 8 160-179 78 80 80 129 133 133 F F F <--- 9 180-199 78 80 80 129 133 133 F T T <--- 10 200-219 79 79 79 130 130 130 F F F 11 220-239 74 74 74 120 120 120 F F F 12 240-259 77 77 77 127 127 127 F F F 13 260-279 79 80 80 130 132 132 F F F <--- 14 280-299 74 80 80 120 132 132 F F F <--- 15 300-319 76 80 80 126 134 134 F F F <--- 16 320-339 79 79 79 130 130 130 F F F 17 340-359 77 77 77 127 127 127 F F F 18 360-379 74 74 74 120 120 120 F F F 19 380-399 65 80 80 105 132 132 F F F <--- 20 400-419 77 77 77 127 127 127 F F F 21 420-439 80 80 80 131 132 132 F F F <--- 22 440-459 79 79 79 130 130 130 F F F 23 460-479 79 79 79 130 130 130 F F F 24 480-499 79 79 79 130 130 130 F F F 25 500-519 79 80 80 130 132 132 F F F <--- 26 520-539 76 80 80 126 132 132 F F F <--- 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 80, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 79 79 79 130 130 130 F F F 2 040-059 81 81 81 135 135 135 F F F 3 060-079 81 81 81 135 135 135 F F F 4 080-099 80 80 80 132 132 132 F F F 5 100-119 76 76 76 125 125 125 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 76 76 76 126 126 126 F F F 8 160-179 80 80 80 133 133 133 F F F 9 180-199 81 81 81 135 135 135 F F F 10 200-219 79 79 79 130 130 130 F F F 11 220-239 74 74 74 120 120 120 F F F 12 240-259 77 77 77 127 127 127 F F F 13 260-279 80 80 80 132 132 132 F F F 14 280-299 80 80 80 132 132 132 F F F 15 300-319 80 80 80 134 134 134 F F F 16 320-339 79 79 79 130 130 130 F F F 17 340-359 77 77 77 127 127 127 F F F 18 360-379 74 74 74 120 120 120 F F F 19 380-399 80 80 80 132 132 132 F F F 20 400-419 77 77 77 127 127 127 F F F 21 420-439 80 80 80 132 132 132 F F F 22 440-459 79 79 79 130 130 130 F F F 23 460-479 79 79 79 130 130 130 F F F 24 480-499 79 79 79 130 130 130 F F F 25 500-519 80 80 80 132 132 132 F F F 26 520-539 80 80 80 132 132 132 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 81, Max flush: 135 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.506679403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.514908889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515197653Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515227189Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515235838Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515247217Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Crucible cf8b5a8c-a309-42e0-a2de-18eb491ff741 has session id: 5d402070-ccd9-4f44-9aed-1a984a4e82ff","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515625899Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Upstairs opts: Upstairs UUID: cf8b5a8c-a309-42e0-a2de-18eb491ff741, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515683151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Crucible stats registered with UUID: cf8b5a8c-a309-42e0-a2de-18eb491ff741","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.51569351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:49.515702569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515839051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515868388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.515878956Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516012709Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516029337Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"cf8b5a8c-a309-42e0-a2de-18eb491ff741 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516047535Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516058264Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"0","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516067862Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"1","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516077351Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"2","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.51608674Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516096699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516106478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:62084)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516217104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:64071)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516259908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:56280)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516270447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516824057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"2","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516849343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"1","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.516862932Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"0","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517217817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[0]R flush_numbers[0..12]: [95, 130, 135, 135, 132, 125, 130, 126, 133, 135, 130, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517245473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[0]R generation[0..12]: [59, 79, 81, 81, 80, 76, 79, 76, 80, 81, 79, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517256492Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517274369Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[1]R flush_numbers[0..12]: [95, 130, 135, 135, 132, 125, 130, 126, 133, 135, 130, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517284998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[1]R generation[0..12]: [59, 79, 81, 81, 80, 76, 79, 76, 80, 81, 79, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517295357Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517306135Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[2]R flush_numbers[0..12]: [95, 130, 135, 135, 132, 125, 130, 126, 133, 135, 130, 120]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517316644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[2]R generation[0..12]: [59, 79, 81, 81, 80, 76, 79, 76, 80, 81, 79, 74]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517326953Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517337691Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.51734694Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517356669Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517366058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517375507Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"":"downstairs","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517385165Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517393834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"0","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517403433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"1","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517414652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"client":"2","":"downstairs","session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.51742443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517433509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"cf8b5a8c-a309-42e0-a2de-18eb491ff741 is now active with session: 5d402070-ccd9-4f44-9aed-1a984a4e82ff","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517443338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517451997Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"session_id":"5d402070-ccd9-4f44-9aed-1a984a4e82ff"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517461026Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Activated sub_volume cf8b5a8c-a309-42e0-a2de-18eb491ff741","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517468905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517476914Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.517780335Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"task":"crutest"} 01/30 Write block 161 len 16384 data: 8 8 10 11 02/30 Write block 352 len 4096 data: 8 03/30 Flush 04/30 Write block 480 len 4096 data: 6 05/30 Write block 51 len 28672 data: 11 9 9 9 9 10 12 06/30 Read block 491 len 32768 07/30 Write block 197 len 36864 data: 5 6 7 7 7 7 8 8 9 08/30 Read block 66 len 4096 09/30 Flush 10/30 Write block 27 len 24576 data: 6 6 6 6 6 5 11/30 Write block 478 len 28672 data: 5 6 7 5 5 7 7 12/30 Flush 13/30 Write block 187 len 4096 data: 6 14/30 Read block 332 len 24576 15/30 Write block 499 len 40960 data: 4 4 3 4 5 6 7 6 7 9 16/30 Write block 44 len 24576 data: 9 7 9 9 9 12 17/30 Write block 414 len 32768 data: 6 6 8 8 9 11 11 13 18/30 Read block 118 len 36864 19/30 Flush 20/30 Write block 183 len 28672 data: 11 8 6 7 7 6 7 21/30 Write block 239 len 16384 data: 6 7 5 6 22/30 Write block 204 len 12288 data: 9 10 9 23/30 Read block 287 len 28672 24/30 Write block 365 len 32768 data: 8 8 8 6 7 7 5 5 25/30 Read block 174 len 32768 26/30 Read block 102 len 40960 27/30 Read block 463 len 32768 28/30 Read block 422 len 16384 29/30 Read block 533 len 20480 30/30 Read block 262 len 36864 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:54:49.526371436Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:54:49.526556162Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1501,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 82 79 82 138 130 138 F F F <--- 2 040-059 82 81 82 139 135 139 F F F <--- 3 060-079 81 81 81 135 135 135 F F F 4 080-099 80 80 80 132 132 132 F F F 5 100-119 76 76 76 125 125 125 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 76 76 76 126 126 126 F F F 8 160-179 82 80 82 136 133 136 F T F <--- 9 180-199 82 81 82 139 135 139 T F T <--- 10 200-219 82 79 82 137 130 137 T F T <--- 11 220-239 74 74 74 120 120 120 T F T <--- 12 240-259 77 77 77 127 127 127 T F T <--- 13 260-279 80 80 80 132 132 132 F F F 14 280-299 80 80 80 132 132 132 F F F 15 300-319 80 80 80 134 134 134 F F F 16 320-339 79 79 79 130 130 130 F F F 17 340-359 82 77 82 136 127 136 F T F <--- 18 360-379 74 74 74 120 120 120 T F T <--- 19 380-399 80 80 80 132 132 132 F F F 20 400-419 82 77 82 139 127 139 F F F <--- 21 420-439 82 80 82 139 132 139 F F F <--- 22 440-459 79 79 79 130 130 130 F F F 23 460-479 82 79 82 138 130 138 F F F <--- 24 480-499 82 79 82 139 130 139 F F F <--- 25 500-519 82 80 82 139 132 139 F F F <--- 26 520-539 80 80 80 132 132 132 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 82, Max flush: 139 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 82 82 82 138 138 138 F F F 2 040-059 82 82 82 139 139 139 F F F 3 060-079 81 81 81 135 135 135 F F F 4 080-099 80 80 80 132 132 132 F F F 5 100-119 76 76 76 125 125 125 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 76 76 76 126 126 126 F F F 8 160-179 82 82 82 136 136 136 F F F 9 180-199 83 83 83 140 140 140 F F F 10 200-219 83 83 83 140 140 140 F F F 11 220-239 83 83 83 140 140 140 F F F 12 240-259 83 83 83 140 140 140 F F F 13 260-279 80 80 80 132 132 132 F F F 14 280-299 80 80 80 132 132 132 F F F 15 300-319 80 80 80 134 134 134 F F F 16 320-339 79 79 79 130 130 130 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 83 83 83 140 140 140 F F F 19 380-399 80 80 80 132 132 132 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 79 79 79 130 130 130 F F F 23 460-479 82 82 82 138 138 138 F F F 24 480-499 82 82 82 139 139 139 F F F 25 500-519 82 82 82 139 139 139 F F F 26 520-539 80 80 80 132 132 132 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 66 66 66 106 106 106 F F F Max gen: 83, Max flush: 140 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.77860869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.786596797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.786881231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.786905908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.786914387Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.786925995Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Crucible 3ed6de2b-a3af-40a6-99b5-1ffae9c760ab has session id: e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787315746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Upstairs opts: Upstairs UUID: 3ed6de2b-a3af-40a6-99b5-1ffae9c760ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787409834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Crucible stats registered with UUID: 3ed6de2b-a3af-40a6-99b5-1ffae9c760ab","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.78743675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:03.787446959Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787575623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787606489Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787617318Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787730073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.78775558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"3ed6de2b-a3af-40a6-99b5-1ffae9c760ab active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787772698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787781557Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"0","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787791056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"1","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787800774Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"2","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787810473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787820282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787830181Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40745)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787870325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58678)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787927908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61107)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.787952515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.788597663Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"2","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.788609612Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"1","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.78862043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"0","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.788977525Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[0]R flush_numbers[0..12]: [95, 138, 139, 135, 132, 125, 130, 126, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.78902088Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[0]R generation[0..12]: [59, 82, 82, 81, 80, 76, 79, 76, 82, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789031768Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789042647Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[1]R flush_numbers[0..12]: [95, 138, 139, 135, 132, 125, 130, 126, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789053105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[1]R generation[0..12]: [59, 82, 82, 81, 80, 76, 79, 76, 82, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789063384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789074043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[2]R flush_numbers[0..12]: [95, 138, 139, 135, 132, 125, 130, 126, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789084402Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[2]R generation[0..12]: [59, 82, 82, 81, 80, 76, 79, 76, 82, 83, 83, 83]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.78909672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789107699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789117007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789126616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789136005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789145204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"":"downstairs","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789154743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789163411Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"0","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.78917296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"1","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789182649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"client":"2","":"downstairs","session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789192388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789201577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"3ed6de2b-a3af-40a6-99b5-1ffae9c760ab is now active with session: e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789211355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789220034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"session_id":"e7c9532f-ab7e-4f46-aa54-e9d6c590ef4f"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789228863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Activated sub_volume 3ed6de2b-a3af-40a6-99b5-1ffae9c760ab","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789236772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789244691Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.789538764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"task":"crutest"} 01/30 Write block 107 len 24576 data: 9 11 11 10 9 8 02/30 Write block 329 len 36864 data: 7 8 4 6 7 8 9 9 9 03/30 Read block 143 len 16384 04/30 Write block 232 len 28672 data: 5 5 4 6 7 6 8 05/30 Write block 245 len 24576 data: 7 6 7 10 9 4 06/30 Read block 397 len 8192 07/30 Read block 571 len 36864 08/30 Write block 244 len 40960 data: 5 8 7 8 11 10 5 6 7 7 09/30 Write block 581 len 20480 data: 9 8 8 6 5 10/30 Write block 96 len 20480 data: 6 6 7 6 7 11/30 Write block 360 len 28672 data: 7 7 8 9 9 9 9 12/30 Read block 37 len 24576 13/30 Flush 14/30 Flush 15/30 Write block 204 len 28672 data: 10 11 10 11 11 9 7 16/30 Flush 17/30 Write block 158 len 16384 data: 9 7 8 9 18/30 Flush 19/30 Write block 460 len 28672 data: 7 6 8 11 7 8 6 20/30 Flush 21/30 Write block 30 len 36864 data: 7 7 6 6 6 5 5 5 4 22/30 Write block 376 len 40960 data: 8 8 8 8 7 6 5 5 5 4 23/30 Write block 266 len 32768 data: 5 5 6 7 6 5 5 5 24/30 Write block 326 len 20480 data: 5 7 7 8 9 25/30 Read block 236 len 20480 26/30 Read block 296 len 20480 27/30 Write block 287 len 12288 data: 9 6 7 28/30 Write block 163 len 12288 data: 11 12 8 29/30 Write block 230 len 32768 data: 5 5 6 6 5 7 8 7 30/30 Read block 75 len 20480 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:03.798000961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:03.798169919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1510,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 82 82 82 138 138 138 T F T <--- 2 040-059 82 82 82 139 139 139 F F F 3 060-079 81 81 81 135 135 135 F F F 4 080-099 84 80 84 141 132 141 F F F <--- 5 100-119 84 76 84 141 125 141 F T F <--- 6 120-139 79 79 79 130 130 130 F F F 7 140-159 84 76 84 144 126 144 F F F <--- 8 160-179 84 82 84 144 136 144 T F T <--- 9 180-199 83 83 83 140 140 140 F F F 10 200-219 84 83 84 143 140 143 F F F <--- 11 220-239 84 83 84 141 140 141 T F T <--- 12 240-259 84 83 84 141 140 141 F F F <--- 13 260-279 80 80 80 132 132 132 T F T <--- 14 280-299 80 80 80 132 132 132 T F T <--- 15 300-319 80 80 80 134 134 134 F F F 16 320-339 84 79 84 141 130 141 T T T <--- 17 340-359 82 82 82 136 136 136 F F F 18 360-379 84 83 84 141 140 141 T F T <--- 19 380-399 80 80 80 132 132 132 T F T <--- 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 79 79 79 130 130 130 F F F 23 460-479 84 82 84 145 138 145 F F F <--- 24 480-499 82 82 82 139 139 139 F F F 25 500-519 82 82 82 139 139 139 F F F 26 520-539 80 80 80 132 132 132 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 84 66 84 141 106 141 F F F <--- Max gen: 84, Max flush: 145 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 95 95 95 F F F 1 020-039 85 85 85 146 146 146 F F F 2 040-059 82 82 82 139 139 139 F F F 3 060-079 81 81 81 135 135 135 F F F 4 080-099 84 84 84 141 141 141 F F F 5 100-119 84 84 84 141 141 141 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 83 83 83 140 140 140 F F F 10 200-219 84 84 84 143 143 143 F F F 11 220-239 85 85 85 146 146 146 F F F 12 240-259 84 84 84 141 141 141 F F F 13 260-279 85 85 85 146 146 146 F F F 14 280-299 85 85 85 146 146 146 F F F 15 300-319 80 80 80 134 134 134 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 82 82 82 139 139 139 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 79 79 79 130 130 130 F F F 23 460-479 84 84 84 145 145 145 F F F 24 480-499 82 82 82 139 139 139 F F F 25 500-519 82 82 82 139 139 139 F F F 26 520-539 80 80 80 132 132 132 F F F 27 540-559 70 70 70 114 114 114 F F F 28 560-579 68 68 68 110 110 110 F F F 29 580-599 84 84 84 141 141 141 F F F Max gen: 85, Max flush: 146 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.059299466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067193915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067471519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067505585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067514894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067526692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Crucible b577df6a-c2c1-46bd-9875-0e6510f5e75f has session id: 9aa01bc1-4c0a-4a9b-91c1-04064a57e222","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067934261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Upstairs opts: Upstairs UUID: b577df6a-c2c1-46bd-9875-0e6510f5e75f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067955118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Crucible stats registered with UUID: b577df6a-c2c1-46bd-9875-0e6510f5e75f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.067964807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:18.067973866Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068116568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068147864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068158772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068299114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068314732Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"b577df6a-c2c1-46bd-9875-0e6510f5e75f active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068357027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068366396Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"0","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068375915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"1","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068385533Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"2","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068394852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068404911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.06841478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"ds_connection connected from Ok(127.0.0.1:56969)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068475562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"ds_connection connected from Ok(127.0.0.1:50597)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068486361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"ds_connection connected from Ok(127.0.0.1:47232)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.068496409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069142037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"0","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069186322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"1","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069210039Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"2","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069500682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[0]R flush_numbers[0..12]: [95, 146, 139, 135, 141, 141, 130, 144, 146, 140, 143, 146]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069521169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[0]R generation[0..12]: [59, 85, 82, 81, 84, 84, 79, 84, 85, 83, 84, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069531658Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069542587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[1]R flush_numbers[0..12]: [95, 146, 139, 135, 141, 141, 130, 144, 146, 140, 143, 146]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069553115Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[1]R generation[0..12]: [59, 85, 82, 81, 84, 84, 79, 84, 85, 83, 84, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069563334Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069574023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[2]R flush_numbers[0..12]: [95, 146, 139, 135, 141, 141, 130, 144, 146, 140, 143, 146]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069584461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[2]R generation[0..12]: [59, 85, 82, 81, 84, 84, 79, 84, 85, 83, 84, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.06959462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069605689Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069614967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069624606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069633885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069643224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"":"downstairs","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069654482Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069663111Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"0","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.06967314Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"1","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069682789Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"client":"2","":"downstairs","session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069692398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069701447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"b577df6a-c2c1-46bd-9875-0e6510f5e75f is now active with session: 9aa01bc1-4c0a-4a9b-91c1-04064a57e222","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069711255Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069719804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"session_id":"9aa01bc1-4c0a-4a9b-91c1-04064a57e222"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069728753Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Activated sub_volume b577df6a-c2c1-46bd-9875-0e6510f5e75f","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069736752Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.069744691Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.070043143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"task":"crutest"} 01/30 Write block 10 len 12288 data: 5 8 9 02/30 Flush 03/30 Write block 314 len 8192 data: 6 6 04/30 Flush 05/30 Write block 54 len 20480 data: 10 10 11 13 12 06/30 Write block 460 len 24576 data: 8 7 9 12 8 9 07/30 Read block 507 len 12288 08/30 Read block 174 len 16384 09/30 Flush 10/30 Write block 67 len 20480 data: 6 4 5 5 10 11/30 Write block 490 len 40960 data: 2 3 4 5 3 3 3 3 3 5 12/30 Write block 564 len 28672 data: 4 5 5 5 4 4 3 13/30 Write block 531 len 24576 data: 9 9 10 9 9 9 14/30 Write block 206 len 12288 data: 11 12 12 15/30 Write block 84 len 40960 data: 5 7 8 8 10 10 9 7 4 4 16/30 Write block 46 len 24576 data: 10 10 10 13 10 12 17/30 Read block 591 len 28672 18/30 Read block 112 len 8192 19/30 Flush 20/30 Write block 442 len 16384 data: 6 7 7 8 21/30 Write block 14 len 36864 data: 7 6 5 5 5 4 3 3 3 22/30 Read block 479 len 8192 23/30 Flush 24/30 Write block 267 len 20480 data: 6 7 8 7 6 25/30 Read block 166 len 4096 26/30 Read block 549 len 40960 27/30 Read block 347 len 24576 28/30 Write block 549 len 20480 data: 5 6 5 6 6 29/30 Write block 44 len 28672 data: 10 8 11 11 11 14 11 30/30 Write block 411 len 28672 data: 3 5 5 7 7 9 9 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 5 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 5 Done Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 3 31 Done 24 0 23 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1003)) Some(JobId(1022)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:18.07779211Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:18.077985596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1519,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 147 151 F F F <--- 1 020-039 86 85 86 151 146 151 F F F <--- 2 040-059 86 82 86 150 139 150 T F T <--- 3 060-079 86 81 86 150 135 150 F F F <--- 4 080-099 86 84 86 150 141 150 F F F <--- 5 100-119 84 84 84 141 141 141 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 83 83 83 140 140 140 F F F 10 200-219 86 84 86 150 143 150 F F F <--- 11 220-239 85 85 85 146 146 146 F F F 12 240-259 84 84 84 141 141 141 F F F 13 260-279 85 85 85 146 146 146 T F T <--- 14 280-299 85 85 85 146 146 146 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 82 82 82 139 139 139 T F T <--- 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 79 86 151 130 151 F F F <--- 23 460-479 86 84 86 149 145 149 F F F <--- 24 480-499 86 82 86 150 139 150 F F F <--- 25 500-519 82 82 82 139 139 139 F F F 26 520-539 86 80 86 150 132 150 F F F <--- 27 540-559 70 70 70 114 114 114 T F T <--- 28 560-579 86 68 86 150 110 150 F F F <--- 29 580-599 84 84 84 141 141 141 F F F Max gen: 86, Max flush: 151 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 87 87 87 152 152 152 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 84 84 84 141 141 141 F F F 6 120-139 79 79 79 130 130 130 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 83 83 83 140 140 140 F F F 10 200-219 86 86 86 150 150 150 F F F 11 220-239 85 85 85 146 146 146 F F F 12 240-259 84 84 84 141 141 141 F F F 13 260-279 87 87 87 152 152 152 F F F 14 280-299 85 85 85 146 146 146 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 87 87 87 152 152 152 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 86 86 151 151 151 F F F 23 460-479 86 86 86 149 149 149 F F F 24 480-499 86 86 86 150 150 150 F F F 25 500-519 82 82 82 139 139 139 F F F 26 520-539 86 86 86 150 150 150 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 84 84 84 141 141 141 F F F Max gen: 87, Max flush: 152 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.331553221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.339484926Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.339773949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.339808754Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.339817973Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.339832142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Crucible e14574b5-ab4c-4c26-8791-0aa850a582be has session id: e0101972-54a3-458e-a2a7-05138f9d432c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340191536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Upstairs opts: Upstairs UUID: e14574b5-ab4c-4c26-8791-0aa850a582be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340211063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Crucible stats registered with UUID: e14574b5-ab4c-4c26-8791-0aa850a582be","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340223252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:32.340232221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340366014Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340382842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34039271Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34055157Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340575147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"e14574b5-ab4c-4c26-8791-0aa850a582be active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340592035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340600804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"0","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340610393Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"1","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340619812Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"2","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34062926Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340640909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340650918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63263)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340701981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51442)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340726298Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49464)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.340737427Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341376626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"0","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34142207Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"1","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341519577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"2","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341785814Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[0]R flush_numbers[0..12]: [151, 151, 152, 150, 150, 141, 130, 144, 146, 140, 150, 146]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34181885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[0]R generation[0..12]: [86, 86, 87, 86, 86, 84, 79, 84, 85, 83, 86, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341829948Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341840797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[1]R flush_numbers[0..12]: [151, 151, 152, 150, 150, 141, 130, 144, 146, 140, 150, 146]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341851395Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[1]R generation[0..12]: [86, 86, 87, 86, 86, 84, 79, 84, 85, 83, 86, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341861704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341875222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[2]R flush_numbers[0..12]: [151, 151, 152, 150, 150, 141, 130, 144, 146, 140, 150, 146]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341885831Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[2]R generation[0..12]: [86, 86, 87, 86, 86, 84, 79, 84, 85, 83, 86, 85]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34189606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341906878Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341916177Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341925806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341935215Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341944444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"":"downstairs","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341954092Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341962801Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"0","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.34197228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"1","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341981769Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"client":"2","":"downstairs","session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.341991368Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342000517Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"e14574b5-ab4c-4c26-8791-0aa850a582be is now active with session: e0101972-54a3-458e-a2a7-05138f9d432c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342010685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342021014Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"session_id":"e0101972-54a3-458e-a2a7-05138f9d432c"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342029993Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Activated sub_volume e14574b5-ab4c-4c26-8791-0aa850a582be","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342038132Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342046321Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.342334254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"task":"crutest"} 01/30 Write block 409 len 20480 data: 4 3 4 6 6 02/30 Read block 427 len 16384 03/30 Read block 83 len 8192 04/30 Read block 496 len 8192 05/30 Read block 362 len 12288 06/30 Read block 313 len 32768 07/30 Write block 50 len 28672 data: 12 13 10 10 11 11 12 08/30 Read block 46 len 16384 09/30 Read block 23 len 12288 10/30 Read block 85 len 32768 11/30 Read block 339 len 12288 12/30 Write block 280 len 32768 data: 9 9 9 11 7 8 9 10 13/30 Write block 121 len 24576 data: 6 7 8 8 9 9 14/30 Write block 516 len 24576 data: 11 10 7 8 8 8 15/30 Read block 236 len 16384 16/30 Read block 554 len 16384 17/30 Read block 249 len 36864 18/30 Write block 234 len 4096 data: 6 19/30 Read block 69 len 28672 20/30 Read block 226 len 12288 21/30 Write block 115 len 8192 data: 7 6 22/30 Read block 85 len 36864 23/30 Flush 24/30 Write block 122 len 24576 data: 8 9 9 10 10 9 25/30 Read block 220 len 28672 26/30 Read block 535 len 40960 27/30 Read block 143 len 20480 28/30 Read block 408 len 32768 29/30 Read block 21 len 40960 30/30 Write block 84 len 20480 data: 6 8 9 9 11 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 18 20 Done 29 29 12 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:32.351207959Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:32.351386936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1528,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 88 88 87 153 153 152 F F T <--- 3 060-079 86 86 86 150 150 150 F F F 4 080-099 86 86 86 150 150 150 T T F <--- 5 100-119 88 88 84 153 153 141 F F F <--- 6 120-139 88 88 79 153 153 130 T T F <--- 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 83 83 83 140 140 140 F F F 10 200-219 86 86 86 150 150 150 F F F 11 220-239 88 88 85 153 153 146 F F F <--- 12 240-259 84 84 84 141 141 141 F F F 13 260-279 87 87 87 152 152 152 F F F 14 280-299 88 88 85 153 153 146 F F T <--- 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 88 88 87 153 153 152 F F T <--- 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 86 86 151 151 151 F F F 23 460-479 86 86 86 149 149 149 F F F 24 480-499 86 86 86 150 150 150 F F F 25 500-519 88 88 82 153 153 139 F F F <--- 26 520-539 88 88 86 153 153 150 F F F <--- 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 84 84 84 141 141 141 F F F Max gen: 88, Max flush: 153 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 89 89 89 154 154 154 F F F 5 100-119 88 88 88 153 153 153 F F F 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 83 83 83 140 140 140 F F F 10 200-219 86 86 86 150 150 150 F F F 11 220-239 88 88 88 153 153 153 F F F 12 240-259 84 84 84 141 141 141 F F F 13 260-279 87 87 87 152 152 152 F F F 14 280-299 88 88 88 153 153 153 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 88 88 88 153 153 153 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 86 86 151 151 151 F F F 23 460-479 86 86 86 149 149 149 F F F 24 480-499 86 86 86 150 150 150 F F F 25 500-519 88 88 88 153 153 153 F F F 26 520-539 88 88 88 153 153 153 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 84 84 84 141 141 141 F F F Max gen: 89, Max flush: 154 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.596717816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.604672037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.604947442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.604972619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.604981238Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.604992727Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Crucible 8a7b94df-4b04-49bb-9f0a-70ff833251e0 has session id: dc04b8db-65e2-4451-bd24-696211e72959","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605354591Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Upstairs opts: Upstairs UUID: 8a7b94df-4b04-49bb-9f0a-70ff833251e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605383427Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Crucible stats registered with UUID: 8a7b94df-4b04-49bb-9f0a-70ff833251e0","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605394416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:46.605403515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605554336Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605584842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60559588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605713865Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605729413Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"8a7b94df-4b04-49bb-9f0a-70ff833251e0 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605745141Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60575379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"0","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605763339Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"1","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605772738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"2","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605782217Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605792015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605802124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"ds_connection connected from Ok(127.0.0.1:41259)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605848248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"ds_connection connected from Ok(127.0.0.1:51134)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605873315Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"ds_connection connected from Ok(127.0.0.1:38852)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.605975912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60654537Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"1","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.606566127Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"2","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.606577186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"0","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60693651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[0]R flush_numbers[0..12]: [151, 151, 153, 150, 154, 153, 154, 144, 146, 140, 150, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.606966906Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[0]R generation[0..12]: [86, 86, 88, 86, 89, 88, 89, 84, 85, 83, 86, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.606977735Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.606988664Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[1]R flush_numbers[0..12]: [151, 151, 153, 150, 154, 153, 154, 144, 146, 140, 150, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607010021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[1]R generation[0..12]: [86, 86, 88, 86, 89, 88, 89, 84, 85, 83, 86, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60702022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607031058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[2]R flush_numbers[0..12]: [151, 151, 153, 150, 154, 153, 154, 144, 146, 140, 150, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607041617Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[2]R generation[0..12]: [86, 86, 88, 86, 89, 88, 89, 84, 85, 83, 86, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607051856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607062794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607072113Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607081912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60709277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607102219Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"":"downstairs","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607111768Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607120467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"0","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607129946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"1","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607139434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"client":"2","":"downstairs","session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607149243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607158312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"8a7b94df-4b04-49bb-9f0a-70ff833251e0 is now active with session: dc04b8db-65e2-4451-bd24-696211e72959","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607168191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60717691Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"session_id":"dc04b8db-65e2-4451-bd24-696211e72959"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607185889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Activated sub_volume 8a7b94df-4b04-49bb-9f0a-70ff833251e0","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607193868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.607201957Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.60749135Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"task":"crutest"} 01/30 Write block 205 len 32768 data: 12 12 13 13 10 8 7 8 02/30 Flush 03/30 Write block 479 len 40960 data: 7 8 6 6 8 8 8 6 6 4 04/30 Write block 536 len 8192 data: 10 8 05/30 Read block 565 len 16384 06/30 Read block 141 len 8192 07/30 Read block 145 len 24576 08/30 Read block 321 len 40960 09/30 Write block 194 len 8192 data: 9 8 10/30 Read block 194 len 40960 11/30 Read block 25 len 24576 12/30 Read block 300 len 4096 13/30 Read block 473 len 28672 14/30 Read block 54 len 20480 15/30 Read block 351 len 4096 16/30 Flush 17/30 Write block 277 len 40960 data: 8 8 8 10 10 10 12 8 9 10 18/30 Write block 281 len 4096 data: 11 19/30 Read block 586 len 12288 20/30 Read block 342 len 32768 21/30 Flush 22/30 Write block 181 len 16384 data: 8 8 12 9 23/30 Read block 508 len 40960 24/30 Flush 25/30 Write block 585 len 28672 data: 6 6 6 6 5 6 7 26/30 Write block 508 len 32768 data: 10 7 7 7 8 6 8 10 27/30 Read block 485 len 40960 28/30 Write block 189 len 20480 data: 8 7 7 9 11 29/30 Read block 479 len 32768 30/30 Read block 427 len 8192 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 24 24 Done 28 28 4 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:55:46.616489179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:55:46.616652628Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1537,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 89 89 89 154 154 154 F F F 5 100-119 88 88 88 153 153 153 F F F 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 90 90 83 158 158 140 T T F <--- 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 153 153 153 F F F 12 240-259 84 84 84 141 141 141 F F F 13 260-279 90 90 87 157 157 152 F F F <--- 14 280-299 90 90 88 157 157 153 F F F <--- 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 88 88 88 153 153 153 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 86 86 151 151 151 F F F 23 460-479 90 90 86 156 156 149 F F T <--- 24 480-499 90 90 86 156 156 150 F F T <--- 25 500-519 88 88 88 153 153 153 T T F <--- 26 520-539 90 90 88 156 156 153 F F T <--- 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 84 84 84 141 141 141 T T F <--- Max gen: 90, Max flush: 158 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 89 89 89 154 154 154 F F F 5 100-119 88 88 88 153 153 153 F F F 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 91 91 91 159 159 159 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 153 153 153 F F F 12 240-259 84 84 84 141 141 141 F F F 13 260-279 90 90 90 157 157 157 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 F F F 17 340-359 82 82 82 136 136 136 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 88 88 88 153 153 153 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 86 86 151 151 151 F F F 23 460-479 90 90 90 156 156 156 F F F 24 480-499 90 90 90 156 156 156 F F F 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 91 91 91 159 159 159 F F F Max gen: 91, Max flush: 159 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.860776671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.868862526Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869175516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869199073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869209922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.86922134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Crucible 979c7985-7894-447b-b41a-b8f5b62819f9 has session id: d26d1de8-428c-4802-a760-cad2edf40aa9","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869601952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Upstairs opts: Upstairs UUID: 979c7985-7894-447b-b41a-b8f5b62819f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869621839Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Crucible stats registered with UUID: 979c7985-7894-447b-b41a-b8f5b62819f9","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869633998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:00.869642897Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869795967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869827003Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869837942Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869980384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.869997202Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"979c7985-7894-447b-b41a-b8f5b62819f9 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870020849Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870030028Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"0","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870039516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"1","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870051065Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"2","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870060554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870070372Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870080221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58708)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870129345Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54220)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870156342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34493)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870232122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.87079696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"0","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870813508Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"1","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.870824347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"2","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871181701Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[0]R flush_numbers[0..12]: [151, 151, 153, 150, 154, 153, 154, 144, 146, 159, 155, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871217677Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[0]R generation[0..12]: [86, 86, 88, 86, 89, 88, 89, 84, 85, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871228626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871239484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[1]R flush_numbers[0..12]: [151, 151, 153, 150, 154, 153, 154, 144, 146, 159, 155, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871252303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[1]R generation[0..12]: [86, 86, 88, 86, 89, 88, 89, 84, 85, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871262541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.87127324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[2]R flush_numbers[0..12]: [151, 151, 153, 150, 154, 153, 154, 144, 146, 159, 155, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871283799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[2]R generation[0..12]: [86, 86, 88, 86, 89, 88, 89, 84, 85, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871294087Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871304856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871314185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871323633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871332862Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871342231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"":"downstairs","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.87135209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871360809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"0","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871370258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"1","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871379806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"client":"2","":"downstairs","session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871389515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871400004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"979c7985-7894-447b-b41a-b8f5b62819f9 is now active with session: d26d1de8-428c-4802-a760-cad2edf40aa9","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871409823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871418501Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"session_id":"d26d1de8-428c-4802-a760-cad2edf40aa9"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.87142756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Activated sub_volume 979c7985-7894-447b-b41a-b8f5b62819f9","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871435549Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871443698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.871779366Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"task":"crutest"} 01/30 Write block 493 len 8192 data: 6 4 02/30 Write block 88 len 4096 data: 12 03/30 Read block 564 len 8192 04/30 Read block 260 len 8192 05/30 Write block 463 len 20480 data: 13 9 10 7 5 06/30 Read block 445 len 8192 07/30 Read block 179 len 4096 08/30 Read block 419 len 16384 09/30 Read block 524 len 40960 10/30 Write block 401 len 20480 data: 6 7 8 9 7 11/30 Write block 119 len 4096 data: 6 12/30 Flush 13/30 Write block 77 len 40960 data: 12 12 12 9 6 6 5 7 9 10 14/30 Flush 15/30 Write block 321 len 4096 data: 4 16/30 Read block 547 len 8192 17/30 Read block 81 len 32768 18/30 Read block 430 len 40960 19/30 Write block 481 len 20480 data: 7 7 9 9 9 20/30 Read block 478 len 40960 21/30 Read block 143 len 40960 22/30 Write block 451 len 32768 data: 7 7 10 11 8 8 8 8 23/30 Read block 518 len 12288 24/30 Write block 347 len 4096 data: 8 25/30 Read block 192 len 20480 26/30 Read block 377 len 4096 27/30 Read block 454 len 32768 28/30 Read block 368 len 36864 29/30 Write block 258 len 28672 data: 6 6 5 4 5 5 7 30/30 Write block 320 len 8192 data: 6 5 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:00.880019971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:00.880203107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1546,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 86 92 161 150 161 F F F <--- 4 080-099 92 89 92 161 154 161 F F F <--- 5 100-119 92 88 92 160 153 160 F F F <--- 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 91 91 91 159 159 159 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 153 153 153 F F F 12 240-259 84 84 84 141 141 141 T F T <--- 13 260-279 90 90 90 157 157 157 T F T <--- 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 85 85 85 146 146 146 T F T <--- 17 340-359 82 82 82 136 136 136 T F T <--- 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 88 92 160 153 160 F F F <--- 21 420-439 82 82 82 139 139 139 F F F 22 440-459 86 86 86 151 151 151 T F T <--- 23 460-479 92 90 92 160 156 160 F F F <--- 24 480-499 92 90 92 160 156 160 T F T <--- 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 91 91 91 159 159 159 F F F Max gen: 92, Max flush: 161 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F F F 1 020-039 86 86 86 151 151 151 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 92 92 92 161 161 161 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F F F 9 180-199 91 91 91 159 159 159 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 153 153 153 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 93 93 93 162 162 162 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 93 93 93 162 162 162 F F F 17 340-359 93 93 93 162 162 162 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F F F 29 580-599 91 91 91 159 159 159 F F F Max gen: 93, Max flush: 162 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.135145208Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:15.142094687Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.142144411Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.148939789Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149193367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149220113Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149228452Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149239891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Crucible 6d4855b1-af2a-43ad-8756-636719ab326b has session id: 68bc1185-caec-451e-95ef-a6cc1da94bbd","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149534704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Upstairs opts: Upstairs UUID: 6d4855b1-af2a-43ad-8756-636719ab326b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149556651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Crucible stats registered with UUID: 6d4855b1-af2a-43ad-8756-636719ab326b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149569339Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:15.149578248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149725259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149754306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149765114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149898187Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149909986Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"6d4855b1-af2a-43ad-8756-636719ab326b active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149921454Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149930123Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"0","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149939732Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"1","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149949381Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"2","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.14995892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149968988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.149978887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45635)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.15003734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65147)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.150108881Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:54509)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.150119919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.150842448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"0","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.150864515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"1","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.150908669Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"2","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.15121593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[0]R flush_numbers[0..12]: [151, 151, 153, 161, 161, 160, 154, 144, 146, 159, 155, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151240977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[0]R generation[0..12]: [86, 86, 88, 92, 92, 92, 89, 84, 85, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151251806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151262674Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[1]R flush_numbers[0..12]: [151, 151, 153, 161, 161, 160, 154, 144, 146, 159, 155, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151273103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[1]R generation[0..12]: [86, 86, 88, 92, 92, 92, 89, 84, 85, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151283292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.15129409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[2]R flush_numbers[0..12]: [151, 151, 153, 161, 161, 160, 154, 144, 146, 159, 155, 153]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151304569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[2]R generation[0..12]: [86, 86, 88, 92, 92, 92, 89, 84, 85, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151317467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151328206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151337575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151347004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151356322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151365611Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"":"downstairs","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.15137528Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151383949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"0","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151393758Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"1","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151403616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"client":"2","":"downstairs","session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151413585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151422884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"6d4855b1-af2a-43ad-8756-636719ab326b is now active with session: 68bc1185-caec-451e-95ef-a6cc1da94bbd","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151432783Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151441462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"session_id":"68bc1185-caec-451e-95ef-a6cc1da94bbd"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151450511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Activated sub_volume 6d4855b1-af2a-43ad-8756-636719ab326b","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.15145852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151466539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.151759501Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} 01/30 Write block 582 len 16384 data: 9 9 7 7 02/30 Read block 508 len 4096 03/30 Write block 568 len 8192 data: 5 5 04/30 Write block 210 len 36864 data: 9 8 9 9 9 8 8 7 7 05/30 Read block 428 len 40960 06/30 Read block 410 len 24576 07/30 Read block 263 len 28672 08/30 Write block 170 len 32768 data: 13 12 12 11 11 11 10 9 09/30 Read block 130 len 36864 10/30 Write block 259 len 40960 data: 7 6 5 6 6 8 6 6 7 8 11/30 Read block 528 len 4096 12/30 Read block 578 len 16384 13/30 Read block 198 len 32768 14/30 Write block 351 len 4096 data: 5 15/30 Read block 59 len 24576 16/30 Read block 177 len 36864 17/30 Read block 56 len 8192 18/30 Read block 458 len 16384 19/30 Read block 188 len 16384 20/30 Write block 115 len 8192 data: 8 7 21/30 Read block 455 len 24576 22/30 Read block 231 len 12288 23/30 Read block 271 len 8192 24/30 Write block 17 len 40960 data: 6 6 5 4 4 4 3 3 4 4 25/30 Write block 584 len 28672 data: 8 8 7 7 7 6 7 26/30 Read block 81 len 12288 27/30 Read block 586 len 24576 28/30 Write block 496 len 4096 data: 4 29/30 Read block 232 len 20480 30/30 Write block 218 len 20480 data: 8 7 9 10 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:15.160267952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:15.160447809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1555,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 151 151 151 F T T <--- 1 020-039 86 86 86 151 151 151 F T T <--- 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 92 92 92 161 161 161 F F F 5 100-119 92 92 92 160 160 160 F T T <--- 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 85 85 85 146 146 146 F T T <--- 9 180-199 91 91 91 159 159 159 F F F 10 200-219 90 90 90 155 155 155 F T T <--- 11 220-239 88 88 88 153 153 153 F T T <--- 12 240-259 93 93 93 162 162 162 F T T <--- 13 260-279 93 93 93 162 162 162 F T T <--- 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 93 93 93 162 162 162 F F F 17 340-359 93 93 93 162 162 162 F T T <--- 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 93 93 93 162 162 162 F T T <--- 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 86 86 86 150 150 150 F T T <--- 29 580-599 91 91 91 159 159 159 T T T Max gen: 93, Max flush: 162 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 586a587,588 > 8, > 8, 588a591 > 7, 590,593d592 < 6, < 6, < 5, < 6, 594a594 > 7, diff found after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 163 163 163 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 92 92 92 161 161 161 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 89 89 89 154 154 154 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 94 94 94 163 163 163 F F F 9 180-199 91 91 91 159 159 159 F F F 10 200-219 94 94 94 163 163 163 F F F 11 220-239 94 94 94 163 163 163 F F F 12 240-259 94 94 94 163 163 163 F F F 13 260-279 94 94 94 163 163 163 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 93 93 93 162 162 162 F F F 17 340-359 94 94 94 163 163 163 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 82 82 82 139 139 139 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 87 87 87 152 152 152 F F F 28 560-579 94 94 94 163 163 163 F F F 29 580-599 94 94 94 163 163 163 F F F Max gen: 94, Max flush: 163 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.422585848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:29.429614227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.42966894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.436313677Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.436578714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.436603481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.43661185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.436623408Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Crucible bfaa22fe-b5e8-443c-9c0d-20461262c24a has session id: 941b5ee8-00f4-405c-9627-d81123c239a7","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437020668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Upstairs opts: Upstairs UUID: bfaa22fe-b5e8-443c-9c0d-20461262c24a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437036546Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Crucible stats registered with UUID: bfaa22fe-b5e8-443c-9c0d-20461262c24a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437046344Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:29.437057733Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437206734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437223192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437233081Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437370203Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437382202Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"bfaa22fe-b5e8-443c-9c0d-20461262c24a active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.43739767Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437406319Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"0","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437415808Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"1","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437425246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"2","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437434915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437444844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437454713Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54272)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437530573Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46331)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.43755565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52090)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.437617972Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438217326Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"1","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438230884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"2","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438367087Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"0","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438703594Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[0]R flush_numbers[0..12]: [163, 163, 153, 161, 161, 163, 154, 144, 163, 159, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.43873841Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[0]R generation[0..12]: [94, 94, 88, 92, 92, 94, 89, 84, 94, 91, 94, 94]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438749249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438760057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[1]R flush_numbers[0..12]: [163, 163, 153, 161, 161, 163, 154, 144, 163, 159, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438770506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[1]R generation[0..12]: [94, 94, 88, 92, 92, 94, 89, 84, 94, 91, 94, 94]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438780815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438791553Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[2]R flush_numbers[0..12]: [163, 163, 153, 161, 161, 163, 154, 144, 163, 159, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438801982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[2]R generation[0..12]: [94, 94, 88, 92, 92, 94, 89, 84, 94, 91, 94, 94]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438812171Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438822879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Max found gen is 95","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438832048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Generation requested: 96 >= found:95","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438841577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438852245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438861654Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"":"downstairs","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438871233Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438879932Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"0","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438889391Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"1","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.43889905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"client":"2","":"downstairs","session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438908858Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438917987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"bfaa22fe-b5e8-443c-9c0d-20461262c24a is now active with session: 941b5ee8-00f4-405c-9627-d81123c239a7","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438927786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438936485Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"session_id":"941b5ee8-00f4-405c-9627-d81123c239a7"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438945504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Activated sub_volume bfaa22fe-b5e8-443c-9c0d-20461262c24a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438953363Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.438961462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.439248885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} 01/30 Write block 576 len 4096 data: 9 02/30 Read block 84 len 32768 03/30 Read block 511 len 20480 04/30 Read block 276 len 8192 05/30 Write block 178 len 24576 data: 5 7 9 9 9 13 06/30 Read block 556 len 4096 07/30 Read block 52 len 28672 08/30 Write block 420 len 28672 data: 12 14 12 13 12 12 9 09/30 Read block 301 len 16384 10/30 Read block 146 len 4096 11/30 Write block 124 len 12288 data: 10 11 11 12/30 Write block 347 len 24576 data: 9 7 7 5 6 9 13/30 Read block 303 len 32768 14/30 Read block 427 len 40960 15/30 Read block 469 len 16384 16/30 Write block 130 len 20480 data: 10 14 15 11 11 17/30 Write block 544 len 32768 data: 9 7 5 5 5 6 7 6 18/30 Read block 378 len 12288 19/30 Write block 429 len 32768 data: 6 4 5 5 5 6 6 6 20/30 Read block 547 len 16384 21/30 Write block 127 len 8192 data: 10 8 22/30 Read block 561 len 12288 23/30 Write block 575 len 40960 data: 8 10 9 8 8 9 10 10 10 8 24/30 Read block 60 len 20480 25/30 Flush 26/30 Write block 151 len 24576 data: 8 10 11 11 8 10 27/30 Read block 475 len 24576 28/30 Read block 546 len 12288 29/30 Write block 131 len 16384 data: 15 16 12 12 30/30 Write block 343 len 32768 data: 9 7 10 10 10 8 8 6 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 20 2 2 24 Done 10 28 28 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:29.448476695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:29.448642904Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1567,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 163 163 163 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 92 92 92 161 161 161 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 89 96 96 154 164 164 F T T <--- 7 140-159 84 84 84 144 144 144 F T T <--- 8 160-179 94 96 96 163 164 164 T F F <--- 9 180-199 91 96 96 159 164 164 T F F <--- 10 200-219 94 94 94 163 163 163 F F F 11 220-239 94 94 94 163 163 163 F F F 12 240-259 94 94 94 163 163 163 F F F 13 260-279 94 94 94 163 163 163 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 93 93 93 162 162 162 F F F 17 340-359 94 96 96 163 164 164 F T T <--- 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 82 96 96 139 164 164 T F F <--- 22 440-459 93 93 93 162 162 162 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 87 96 96 152 164 164 F F F <--- 28 560-579 94 96 96 163 164 164 T F F <--- 29 580-599 94 96 96 163 164 164 F F F <--- Max gen: 96, Max flush: 164 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 163 163 163 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 92 92 92 161 161 161 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 97 97 97 165 165 165 F F F 7 140-159 97 97 97 165 165 165 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 96 96 96 164 164 164 F F F 10 200-219 94 94 94 163 163 163 F F F 11 220-239 94 94 94 163 163 163 F F F 12 240-259 94 94 94 163 163 163 F F F 13 260-279 94 94 94 163 163 163 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 93 93 93 162 162 162 F F F 17 340-359 97 97 97 165 165 165 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 92 92 92 160 160 160 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 91 91 91 159 159 159 F F F 26 520-539 90 90 90 156 156 156 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 96 96 96 164 164 164 F F F Max gen: 97, Max flush: 165 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.695830279Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.703815036Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704130576Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704157872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704166151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70417738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Crucible b04f7b96-c95e-4614-b60d-c1a5608d6d0a has session id: 046d1c21-4b19-45f5-b7b6-19ef220dcef0","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704515497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Upstairs opts: Upstairs UUID: b04f7b96-c95e-4614-b60d-c1a5608d6d0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704544463Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Crucible stats registered with UUID: b04f7b96-c95e-4614-b60d-c1a5608d6d0a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704558612Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:43.704567751Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704697544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704713822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704723831Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704871812Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70488446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"b04f7b96-c95e-4614-b60d-c1a5608d6d0a active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704897079Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704905758Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"0","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704917626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"1","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704927105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"2","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704936664Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704976289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.704998116Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40057)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70504872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56927)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.705059878Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41078)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70512586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.705717505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"0","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.705735172Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"1","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70583362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"2","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70614956Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[0]R flush_numbers[0..12]: [163, 163, 153, 161, 161, 163, 165, 165, 164, 164, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706172247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[0]R generation[0..12]: [94, 94, 88, 92, 92, 94, 97, 97, 96, 96, 94, 94]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706183636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706197264Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[1]R flush_numbers[0..12]: [163, 163, 153, 161, 161, 163, 165, 165, 164, 164, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706207912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[1]R generation[0..12]: [94, 94, 88, 92, 92, 94, 97, 97, 96, 96, 94, 94]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706218371Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.7062293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[2]R flush_numbers[0..12]: [163, 163, 153, 161, 161, 163, 165, 165, 164, 164, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706240258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[2]R generation[0..12]: [94, 94, 88, 92, 92, 94, 97, 97, 96, 96, 94, 94]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706250747Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706261696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706271024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706280713Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706289982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706299281Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"":"downstairs","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70630891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706317689Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"0","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706327257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"1","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706336876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"client":"2","":"downstairs","session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706348005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706357104Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"b04f7b96-c95e-4614-b60d-c1a5608d6d0a is now active with session: 046d1c21-4b19-45f5-b7b6-19ef220dcef0","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706366962Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706375551Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"session_id":"046d1c21-4b19-45f5-b7b6-19ef220dcef0"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.70638452Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Activated sub_volume b04f7b96-c95e-4614-b60d-c1a5608d6d0a","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706392549Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706400498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.706680463Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"task":"crutest"} 01/30 Write block 468 len 4096 data: 6 02/30 Read block 123 len 4096 03/30 Read block 248 len 24576 04/30 Read block 68 len 12288 05/30 Write block 159 len 8192 data: 8 9 06/30 Write block 322 len 16384 data: 5 5 4 6 07/30 Read block 517 len 20480 08/30 Read block 43 len 20480 09/30 Write block 213 len 4096 data: 10 10/30 Write block 320 len 24576 data: 7 6 6 6 5 7 11/30 Flush 12/30 Write block 520 len 32768 data: 9 9 6 6 6 8 7 7 13/30 Read block 143 len 20480 14/30 Write block 132 len 40960 data: 17 13 13 10 9 8 9 10 7 11 15/30 Read block 260 len 36864 16/30 Write block 456 len 40960 data: 9 9 9 9 9 8 10 14 10 11 17/30 Read block 114 len 20480 18/30 Write block 211 len 32768 data: 9 10 11 10 9 9 8 9 19/30 Write block 590 len 8192 data: 7 8 20/30 Read block 564 len 28672 21/30 Read block 27 len 12288 22/30 Read block 60 len 8192 23/30 Flush 24/30 Write block 84 len 28672 data: 8 10 11 10 13 11 10 25/30 Read block 33 len 4096 26/30 Write block 511 len 8192 data: 8 9 27/30 Write block 226 len 8192 data: 6 6 28/30 Read block 408 len 40960 29/30 Write block 142 len 16384 data: 11 9 10 8 30/30 Write block 27 len 20480 data: 7 7 7 8 8 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 22 26 Done 28 28 8 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:43.715186644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:43.715360362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1576,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 163 163 163 T T F <--- 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 92 92 92 161 161 161 T T F <--- 5 100-119 94 94 94 163 163 163 F F F 6 120-139 98 98 97 167 167 165 F F F <--- 7 140-159 98 98 97 167 167 165 T T T <--- 8 160-179 98 98 96 166 166 164 F F T <--- 9 180-199 96 96 96 164 164 164 F F F 10 200-219 98 98 94 167 167 163 F F F <--- 11 220-239 94 94 94 163 163 163 T T F <--- 12 240-259 94 94 94 163 163 163 F F F 13 260-279 94 94 94 163 163 163 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 98 98 93 166 166 162 F F T <--- 17 340-359 97 97 97 165 165 165 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 98 98 93 167 167 162 F F F <--- 23 460-479 98 98 92 167 167 160 F F T <--- 24 480-499 94 94 94 163 163 163 F F F 25 500-519 91 91 91 159 159 159 T T F <--- 26 520-539 98 98 90 167 167 156 F F F <--- 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 96 167 167 164 F F F <--- Max gen: 98, Max flush: 167 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 99 99 99 168 168 168 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 99 99 99 168 168 168 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 98 98 98 167 167 167 F F F 7 140-159 99 99 99 168 168 168 F F F 8 160-179 98 98 98 166 166 166 F F F 9 180-199 96 96 96 164 164 164 F F F 10 200-219 98 98 98 167 167 167 F F F 11 220-239 99 99 99 168 168 168 F F F 12 240-259 94 94 94 163 163 163 F F F 13 260-279 94 94 94 163 163 163 F F F 14 280-299 90 90 90 157 157 157 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 97 97 97 165 165 165 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 92 92 92 160 160 160 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 98 98 98 167 167 167 F F F 23 460-479 98 98 98 167 167 167 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 99 99 99 168 168 168 F F F 26 520-539 98 98 98 167 167 167 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 99, Max flush: 168 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.971118439Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979048353Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979342606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979367023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979375712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.97938735Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Crucible 4a0047d8-a8e9-4208-ae0e-60d102893f02 has session id: a8f23a02-3c3d-4e79-b7bd-885962fdb7be","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979800428Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Upstairs opts: Upstairs UUID: 4a0047d8-a8e9-4208-ae0e-60d102893f02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979855251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Crucible stats registered with UUID: 4a0047d8-a8e9-4208-ae0e-60d102893f02","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.979868979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:57.979877958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.9800173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980033138Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980043057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980208216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980224624Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"4a0047d8-a8e9-4208-ae0e-60d102893f02 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980241312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980250121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"0","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.9802598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"1","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980269248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"2","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980278757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980288556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980298345Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"ds_connection connected from Ok(127.0.0.1:34045)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980384544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"ds_connection connected from Ok(127.0.0.1:44565)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.98041231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"ds_connection connected from Ok(127.0.0.1:47749)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.980467143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981114451Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"1","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981143728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"2","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.98120218Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"0","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981495923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[0]R flush_numbers[0..12]: [163, 168, 153, 161, 168, 163, 167, 168, 166, 164, 167, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981513221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[0]R generation[0..12]: [94, 99, 88, 92, 99, 94, 98, 99, 98, 96, 98, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981523619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981534458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[1]R flush_numbers[0..12]: [163, 168, 153, 161, 168, 163, 167, 168, 166, 164, 167, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981544937Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[1]R generation[0..12]: [94, 99, 88, 92, 99, 94, 98, 99, 98, 96, 98, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981555275Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981565984Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[2]R flush_numbers[0..12]: [163, 168, 153, 161, 168, 163, 167, 168, 166, 164, 167, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981576463Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[2]R generation[0..12]: [94, 99, 88, 92, 99, 94, 98, 99, 98, 96, 98, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981586611Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.98160012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981609708Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981619427Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981628556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981637935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"":"downstairs","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981647554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981656213Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"0","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981665791Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"1","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.98167523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"client":"2","":"downstairs","session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981684879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981693828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"4a0047d8-a8e9-4208-ae0e-60d102893f02 is now active with session: a8f23a02-3c3d-4e79-b7bd-885962fdb7be","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981703587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981712315Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"session_id":"a8f23a02-3c3d-4e79-b7bd-885962fdb7be"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981721444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Activated sub_volume 4a0047d8-a8e9-4208-ae0e-60d102893f02","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981729323Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.981737292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.982032545Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"task":"crutest"} 01/30 Write block 247 len 40960 data: 9 12 11 6 7 8 8 6 6 7 02/30 Write block 297 len 4096 data: 8 03/30 Write block 444 len 12288 data: 8 9 9 04/30 Read block 50 len 28672 05/30 Flush 06/30 Write block 399 len 20480 data: 3 5 7 8 9 07/30 Write block 400 len 20480 data: 6 8 9 10 10 08/30 Write block 251 len 8192 data: 8 9 09/30 Write block 425 len 20480 data: 13 10 8 6 7 10/30 Write block 242 len 32768 data: 7 5 6 9 8 10 13 12 11/30 Flush 12/30 Write block 251 len 32768 data: 9 10 9 7 7 8 5 7 13/30 Write block 516 len 4096 data: 12 14/30 Read block 569 len 4096 15/30 Read block 115 len 32768 16/30 Write block 206 len 28672 data: 13 14 14 11 10 10 11 17/30 Write block 459 len 36864 data: 10 10 9 11 15 11 12 8 6 18/30 Write block 12 len 24576 data: 10 7 8 7 6 7 19/30 Write block 519 len 20480 data: 9 10 10 7 7 20/30 Write block 525 len 8192 data: 9 8 21/30 Write block 205 len 40960 data: 13 14 15 15 12 11 11 12 12 11 22/30 Read block 327 len 20480 23/30 Read block 238 len 4096 24/30 Read block 589 len 36864 25/30 Write block 275 len 16384 data: 7 8 9 9 26/30 Read block 93 len 12288 27/30 Read block 42 len 12288 28/30 Read block 452 len 16384 29/30 Read block 71 len 36864 30/30 Read block 178 len 8192 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) None Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:56:57.989867121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:56:57.99003021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1585,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 T F T <--- 1 020-039 99 99 99 168 168 168 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 99 99 99 168 168 168 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 98 98 98 167 167 167 F F F 7 140-159 99 99 99 168 168 168 F F F 8 160-179 98 98 98 166 166 166 F F F 9 180-199 96 96 96 164 164 164 F F F 10 200-219 98 98 98 167 167 167 T F T <--- 11 220-239 99 99 99 168 168 168 F F F 12 240-259 100 94 100 170 163 170 T T T <--- 13 260-279 94 94 94 163 163 163 T F T <--- 14 280-299 100 90 100 169 157 169 F T F <--- 15 300-319 86 86 86 148 148 148 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 97 97 97 165 165 165 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 100 85 100 170 146 170 F F F <--- 20 400-419 100 92 100 170 160 170 F F F <--- 21 420-439 100 96 100 170 164 170 F F F <--- 22 440-459 100 98 100 169 167 169 T T T <--- 23 460-479 98 98 98 167 167 167 T F T <--- 24 480-499 94 94 94 163 163 163 F F F 25 500-519 99 99 99 168 168 168 T F T <--- 26 520-539 98 98 98 167 167 167 T F T <--- 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 100, Max flush: 170 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 171 171 171 F F F 1 020-039 99 99 99 168 168 168 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 F F F 4 080-099 99 99 99 168 168 168 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 98 98 98 167 167 167 F F F 7 140-159 99 99 99 168 168 168 F F F 8 160-179 98 98 98 166 166 166 F F F 9 180-199 96 96 96 164 164 164 F F F 10 200-219 101 101 101 171 171 171 F F F 11 220-239 99 99 99 168 168 168 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 100 100 100 169 169 169 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 97 97 97 165 165 165 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 100 100 100 170 170 170 F F F 20 400-419 100 100 100 170 170 170 F F F 21 420-439 100 100 100 170 170 170 F F F 22 440-459 101 101 101 171 171 171 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 101 101 101 171 171 171 F F F 26 520-539 101 101 101 171 171 171 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 101, Max flush: 171 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.24609416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254016725Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254316417Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254339724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254348343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254359871Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Crucible 848a3883-26ab-456f-acb1-ea9183792f01 has session id: 693c0432-1e48-41fc-b9b6-29ad90fcb812","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.25468377Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Upstairs opts: Upstairs UUID: 848a3883-26ab-456f-acb1-ea9183792f01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254711687Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Crucible stats registered with UUID: 848a3883-26ab-456f-acb1-ea9183792f01","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254723175Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:12.254744912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254898433Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254928089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.254941328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.25507831Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255097758Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"848a3883-26ab-456f-acb1-ea9183792f01 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255121585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255130824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"0","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255140542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"1","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255150051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"2","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.2551596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255169779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255179657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"ds_connection connected from Ok(127.0.0.1:46064)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255266586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"ds_connection connected from Ok(127.0.0.1:59754)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255291763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"ds_connection connected from Ok(127.0.0.1:45772)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.255303022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.25602649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"0","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256065875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"1","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256120448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"2","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256373916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[0]R flush_numbers[0..12]: [171, 168, 153, 161, 168, 163, 167, 168, 166, 164, 171, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256393203Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[0]R generation[0..12]: [101, 99, 88, 92, 99, 94, 98, 99, 98, 96, 101, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256403652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256414571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[1]R flush_numbers[0..12]: [171, 168, 153, 161, 168, 163, 167, 168, 166, 164, 171, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256424979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[1]R generation[0..12]: [101, 99, 88, 92, 99, 94, 98, 99, 98, 96, 101, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256435318Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256446277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[2]R flush_numbers[0..12]: [171, 168, 153, 161, 168, 163, 167, 168, 166, 164, 171, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256456785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[2]R generation[0..12]: [101, 99, 88, 92, 99, 94, 98, 99, 98, 96, 101, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256466984Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256477733Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256487142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.25649667Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256505899Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256515208Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"":"downstairs","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256526367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256535045Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"0","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256544714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"1","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256554443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"client":"2","":"downstairs","session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256564412Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256573521Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"848a3883-26ab-456f-acb1-ea9183792f01 is now active with session: 693c0432-1e48-41fc-b9b6-29ad90fcb812","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256583399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256592038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"session_id":"693c0432-1e48-41fc-b9b6-29ad90fcb812"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256601107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Activated sub_volume 848a3883-26ab-456f-acb1-ea9183792f01","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256609096Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256617085Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.256921676Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"task":"crutest"} 01/30 Write block 532 len 36864 data: 10 11 10 10 11 9 11 15 15 02/30 Write block 122 len 36864 data: 9 10 11 12 12 11 9 11 11 03/30 Write block 412 len 8192 data: 7 7 04/30 Write block 427 len 40960 data: 9 7 8 5 6 6 6 7 7 7 05/30 Read block 150 len 4096 06/30 Read block 502 len 12288 07/30 Write block 155 len 4096 data: 9 08/30 Write block 173 len 4096 data: 12 09/30 Write block 172 len 36864 data: 13 13 12 12 11 10 6 8 10 10/30 Read block 302 len 12288 11/30 Read block 554 len 8192 12/30 Write block 84 len 20480 data: 9 11 12 11 14 13/30 Read block 34 len 40960 14/30 Read block 78 len 12288 15/30 Write block 424 len 20480 data: 13 14 11 10 8 16/30 Read block 526 len 8192 17/30 Write block 347 len 36864 data: 11 9 9 7 7 10 7 6 6 18/30 Write block 72 len 24576 data: 9 8 11 14 13 13 19/30 Write block 340 len 12288 data: 7 8 7 20/30 Read block 269 len 12288 21/30 Read block 483 len 32768 22/30 Read block 514 len 20480 23/30 Write block 336 len 28672 data: 10 10 9 7 8 9 8 24/30 Write block 192 len 36864 data: 10 12 10 9 8 6 7 8 8 25/30 Write block 501 len 16384 data: 4 5 6 7 26/30 Write block 95 len 24576 data: 5 7 7 8 7 8 27/30 Write block 284 len 20480 data: 9 10 11 11 7 28/30 Write block 132 len 20480 data: 18 14 14 11 10 29/30 Read block 368 len 12288 30/30 Read block 1 len 36864 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 9 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:12.264454831Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:12.26461837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1594,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 171 171 171 F F F 1 020-039 99 99 99 168 168 168 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 92 92 92 161 161 161 T F T <--- 4 080-099 99 99 99 168 168 168 T F T <--- 5 100-119 94 94 94 163 163 163 T F T <--- 6 120-139 98 98 98 167 167 167 T F T <--- 7 140-159 99 99 99 168 168 168 T F T <--- 8 160-179 98 98 98 166 166 166 T F T <--- 9 180-199 96 96 96 164 164 164 T F T <--- 10 200-219 101 101 101 171 171 171 T F T <--- 11 220-239 99 99 99 168 168 168 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 100 100 100 169 169 169 T F T <--- 15 300-319 86 86 86 148 148 148 F F F 16 320-339 98 98 98 166 166 166 T F T <--- 17 340-359 97 97 97 165 165 165 T F T <--- 18 360-379 85 85 85 146 146 146 F F F 19 380-399 100 100 100 170 170 170 F F F 20 400-419 100 100 100 170 170 170 T F T <--- 21 420-439 100 100 100 170 170 170 T F T <--- 22 440-459 101 101 101 171 171 171 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 101 101 101 171 171 171 T F T <--- 26 520-539 101 101 101 171 171 171 T F T <--- 27 540-559 96 96 96 164 164 164 T F T <--- 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 101, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 171 171 171 F F F 1 020-039 99 99 99 168 168 168 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 102 102 102 172 172 172 F F F 4 080-099 102 102 102 172 172 172 F F F 5 100-119 102 102 102 172 172 172 F F F 6 120-139 102 102 102 172 172 172 F F F 7 140-159 102 102 102 172 172 172 F F F 8 160-179 102 102 102 172 172 172 F F F 9 180-199 102 102 102 172 172 172 F F F 10 200-219 102 102 102 172 172 172 F F F 11 220-239 99 99 99 168 168 168 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 102 102 102 172 172 172 F F F 15 300-319 86 86 86 148 148 148 F F F 16 320-339 102 102 102 172 172 172 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 100 100 100 170 170 170 F F F 20 400-419 102 102 102 172 172 172 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 101 101 101 171 171 171 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 102 102 102 172 172 172 F F F 26 520-539 102 102 102 172 172 172 F F F 27 540-559 102 102 102 172 172 172 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 102, Max flush: 172 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.529610007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:26.536587892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.536641805Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.543395419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.543651696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.543674933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.543683442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.543694781Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Crucible 1c5faf81-d36f-440c-99ee-f05542b180b3 has session id: 4cec50d6-abd8-45bf-9400-678b32f16baa","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544051306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Upstairs opts: Upstairs UUID: 1c5faf81-d36f-440c-99ee-f05542b180b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544071403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Crucible stats registered with UUID: 1c5faf81-d36f-440c-99ee-f05542b180b3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544084361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:26.54409468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.54425541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544271668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544281566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544431377Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544447895Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"1c5faf81-d36f-440c-99ee-f05542b180b3 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544471082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544480341Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"0","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.54449183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"1","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544501448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"2","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544510877Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544520996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544531055Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"ds_connection connected from Ok(127.0.0.1:48910)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544575149Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"ds_connection connected from Ok(127.0.0.1:44434)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544585968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"ds_connection connected from Ok(127.0.0.1:40321)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.544701613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545261462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"1","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.54527367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"0","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545313915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"2","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545597369Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[0]R flush_numbers[0..12]: [171, 168, 153, 172, 172, 172, 172, 172, 172, 172, 172, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545617387Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[0]R generation[0..12]: [101, 99, 88, 102, 102, 102, 102, 102, 102, 102, 102, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545628176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545641614Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[1]R flush_numbers[0..12]: [171, 168, 153, 172, 172, 172, 172, 172, 172, 172, 172, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545652232Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[1]R generation[0..12]: [101, 99, 88, 102, 102, 102, 102, 102, 102, 102, 102, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545662581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.54567365Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[2]R flush_numbers[0..12]: [171, 168, 153, 172, 172, 172, 172, 172, 172, 172, 172, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545684138Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[2]R generation[0..12]: [101, 99, 88, 102, 102, 102, 102, 102, 102, 102, 102, 99]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545694457Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545705346Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Max found gen is 103","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545714825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Generation requested: 104 >= found:103","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545724623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545733932Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545743291Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"":"downstairs","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.54575278Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545761379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"0","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545770967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"1","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545780416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"client":"2","":"downstairs","session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545791465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545800554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"1c5faf81-d36f-440c-99ee-f05542b180b3 is now active with session: 4cec50d6-abd8-45bf-9400-678b32f16baa","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545810462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545819171Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"session_id":"4cec50d6-abd8-45bf-9400-678b32f16baa"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.54582816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Activated sub_volume 1c5faf81-d36f-440c-99ee-f05542b180b3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545836089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.545844128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.546106815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} 01/30 Write block 88 len 8192 data: 15 12 02/30 Write block 199 len 20480 data: 9 9 8 8 9 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 482 len 4096 data: 8 07/30 Flush 08/30 Write block 49 len 24576 data: 15 13 14 11 11 12 09/30 Write block 384 len 32768 data: 6 5 4 5 5 7 8 8 10/30 Flush 11/30 Write block 457 len 12288 data: 10 10 11 12/30 Read block 253 len 36864 13/30 Write block 502 len 20480 data: 6 7 8 8 7 14/30 Write block 520 len 4096 data: 11 15/30 Read block 19 len 24576 16/30 Write block 182 len 32768 data: 10 14 10 7 8 8 7 9 17/30 Read block 216 len 40960 18/30 Flush 19/30 Write block 302 len 8192 data: 5 7 20/30 Write block 513 len 36864 data: 7 9 11 13 11 8 10 12 11 21/30 Read block 79 len 20480 22/30 Read block 67 len 4096 23/30 Write block 79 len 12288 data: 13 10 7 24/30 Write block 167 len 24576 data: 8 12 13 14 13 14 25/30 Flush 26/30 Write block 146 len 28672 data: 11 10 12 12 10 9 11 27/30 Read block 510 len 12288 28/30 Write block 315 len 12288 data: 7 5 5 29/30 Write block 214 len 32768 data: 12 10 10 9 10 8 10 11 30/30 Write block 303 len 40960 data: 8 7 7 8 9 9 7 7 7 6 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Write 5 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:26.554233574Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:26.554410592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1603,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 171 171 171 F F F 1 020-039 99 99 99 168 168 168 F F F 2 040-059 88 104 104 153 177 177 F F F <--- 3 060-079 102 104 104 172 179 179 F F F <--- 4 080-099 102 104 104 172 179 179 F F F <--- 5 100-119 102 102 102 172 172 172 F F F 6 120-139 102 102 102 172 172 172 F F F 7 140-159 102 102 102 172 172 172 F T T <--- 8 160-179 102 104 104 172 179 179 F F F <--- 9 180-199 102 104 104 172 178 178 F F F <--- 10 200-219 102 104 104 172 173 173 F T T <--- 11 220-239 99 99 99 168 168 168 F T T <--- 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 102 102 102 172 172 172 F F F 15 300-319 86 104 104 148 179 179 F T T <--- 16 320-339 102 102 102 172 172 172 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 100 104 104 170 177 177 F F F <--- 20 400-419 102 102 102 172 172 172 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 101 104 104 171 178 178 F F F <--- 23 460-479 101 101 101 171 171 171 F F F 24 480-499 94 104 104 163 176 176 F F F <--- 25 500-519 102 104 104 172 179 179 F F F <--- 26 520-539 102 104 104 172 179 179 F F F <--- 27 540-559 102 102 102 172 172 172 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 104, Max flush: 179 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 171 171 171 F F F 1 020-039 99 99 99 168 168 168 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 104 104 104 179 179 179 F F F 4 080-099 104 104 104 179 179 179 F F F 5 100-119 102 102 102 172 172 172 F F F 6 120-139 102 102 102 172 172 172 F F F 7 140-159 105 105 105 180 180 180 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 104 104 104 178 178 178 F F F 10 200-219 105 105 105 180 180 180 F F F 11 220-239 105 105 105 180 180 180 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 102 102 102 172 172 172 F F F 15 300-319 105 105 105 180 180 180 F F F 16 320-339 102 102 102 172 172 172 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 85 85 85 146 146 146 F F F 19 380-399 104 104 104 177 177 177 F F F 20 400-419 102 102 102 172 172 172 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 104 104 104 178 178 178 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 104 104 104 176 176 176 F F F 25 500-519 104 104 104 179 179 179 F F F 26 520-539 104 104 104 179 179 179 F F F 27 540-559 102 102 102 172 172 172 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 105, Max flush: 180 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.811017162Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819056933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819404149Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819431815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819440384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819451773Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Crucible f6d5dcec-8cbe-42f1-ba7f-0c7a5234042c has session id: 7f453556-3cbd-43bc-b71f-f00cf149a7cb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819836604Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Upstairs opts: Upstairs UUID: f6d5dcec-8cbe-42f1-ba7f-0c7a5234042c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.81986323Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Crucible stats registered with UUID: f6d5dcec-8cbe-42f1-ba7f-0c7a5234042c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.819873289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:40.819882428Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820044787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820077643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820088492Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820220625Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820248022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"f6d5dcec-8cbe-42f1-ba7f-0c7a5234042c active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.82026105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820269879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"0","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820279548Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"1","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820289136Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"2","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820298615Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820308434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820318253Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64618)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820385134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58675)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820411821Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64244)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.820423039Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821109742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"2","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821123101Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"0","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821137129Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"1","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821492224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[0]R flush_numbers[0..12]: [171, 168, 177, 179, 179, 172, 172, 180, 179, 178, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821517571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[0]R generation[0..12]: [101, 99, 104, 104, 104, 102, 102, 105, 104, 104, 105, 105]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821528719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821539568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[1]R flush_numbers[0..12]: [171, 168, 177, 179, 179, 172, 172, 180, 179, 178, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821550086Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[1]R generation[0..12]: [101, 99, 104, 104, 104, 102, 102, 105, 104, 104, 105, 105]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821560515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821571144Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[2]R flush_numbers[0..12]: [171, 168, 177, 179, 179, 172, 172, 180, 179, 178, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821581722Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[2]R generation[0..12]: [101, 99, 104, 104, 104, 102, 102, 105, 104, 104, 105, 105]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821593921Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821604719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821613918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821623637Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821632936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821642125Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"":"downstairs","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821651654Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821660392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"0","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821670151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"1","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.82167964Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"client":"2","":"downstairs","session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821689469Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821698478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"f6d5dcec-8cbe-42f1-ba7f-0c7a5234042c is now active with session: 7f453556-3cbd-43bc-b71f-f00cf149a7cb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821708436Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821717245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"session_id":"7f453556-3cbd-43bc-b71f-f00cf149a7cb"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821726174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Activated sub_volume f6d5dcec-8cbe-42f1-ba7f-0c7a5234042c","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821734243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.821742302Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.822043723Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"task":"crutest"} 01/30 Write block 379 len 36864 data: 9 8 7 6 6 7 6 5 6 02/30 Write block 413 len 28672 data: 8 8 8 10 10 10 12 03/30 Flush 04/30 Write block 498 len 4096 data: 4 05/30 Read block 125 len 8192 06/30 Read block 498 len 40960 07/30 Write block 88 len 36864 data: 16 13 11 8 5 5 4 6 8 08/30 Write block 295 len 4096 data: 9 09/30 Read block 440 len 4096 10/30 Read block 182 len 4096 11/30 Write block 219 len 28672 data: 9 11 12 10 8 8 7 12/30 Write block 235 len 12288 data: 8 9 8 13/30 Read block 243 len 20480 14/30 Write block 505 len 12288 data: 9 8 8 15/30 Write block 192 len 36864 data: 11 13 11 10 9 7 8 10 10 16/30 Read block 2 len 4096 17/30 Read block 202 len 12288 18/30 Read block 460 len 32768 19/30 Read block 491 len 12288 20/30 Write block 492 len 12288 data: 5 7 5 21/30 Read block 427 len 40960 22/30 Read block 512 len 4096 23/30 Write block 209 len 24576 data: 13 12 12 13 13 13 24/30 Write block 127 len 20480 data: 12 10 12 12 16 25/30 Read block 93 len 12288 26/30 Write block 556 len 12288 data: 4 5 6 27/30 Read block 583 len 16384 28/30 Write block 15 len 24576 data: 8 7 8 7 6 5 29/30 Write block 153 len 28672 data: 12 12 10 11 10 10 9 30/30 Write block 123 len 40960 data: 11 12 13 13 13 11 13 13 17 19 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 3 34 Done 28 1 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) None Some(JobId(1002)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:40.830211857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:40.830392644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1612,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 171 171 171 T F T <--- 1 020-039 99 99 99 168 168 168 T F T <--- 2 040-059 104 104 104 177 177 177 F F F 3 060-079 104 104 104 179 179 179 F F F 4 080-099 104 104 104 179 179 179 T F T <--- 5 100-119 102 102 102 172 172 172 F F F 6 120-139 102 102 102 172 172 172 T F T <--- 7 140-159 105 105 105 180 180 180 T F T <--- 8 160-179 104 104 104 179 179 179 F F F 9 180-199 104 104 104 178 178 178 T F T <--- 10 200-219 105 105 105 180 180 180 T F T <--- 11 220-239 105 105 105 180 180 180 T F T <--- 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 102 102 102 172 172 172 T F T <--- 15 300-319 105 105 105 180 180 180 F F F 16 320-339 102 102 102 172 172 172 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 106 85 106 181 146 181 F T F <--- 19 380-399 106 104 106 181 177 181 F T F <--- 20 400-419 106 102 106 181 172 181 F F F <--- 21 420-439 102 102 102 172 172 172 F F F 22 440-459 104 104 104 178 178 178 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 104 104 104 176 176 176 T F T <--- 25 500-519 104 104 104 179 179 179 T F T <--- 26 520-539 104 104 104 179 179 179 F F F 27 540-559 102 102 102 172 172 172 T F T <--- 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 106, Max flush: 181 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 182 182 182 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 104 104 104 179 179 179 F F F 4 080-099 107 107 107 182 182 182 F F F 5 100-119 102 102 102 172 172 172 F F F 6 120-139 107 107 107 182 182 182 F F F 7 140-159 107 107 107 182 182 182 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 107 107 107 182 182 182 F F F 10 200-219 107 107 107 182 182 182 F F F 11 220-239 107 107 107 182 182 182 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 101 101 101 171 171 171 F F F 14 280-299 107 107 107 182 182 182 F F F 15 300-319 105 105 105 180 180 180 F F F 16 320-339 102 102 102 172 172 172 F F F 17 340-359 102 102 102 172 172 172 F F F 18 360-379 106 106 106 181 181 181 F F F 19 380-399 106 106 106 181 181 181 F F F 20 400-419 106 106 106 181 181 181 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 104 104 104 178 178 178 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 107 107 107 182 182 182 F F F 25 500-519 107 107 107 182 182 182 F F F 26 520-539 104 104 104 179 179 179 F F F 27 540-559 107 107 107 182 182 182 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 107, Max flush: 182 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.094379009Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.102431048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.102712842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.102736739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.102745278Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.102756706Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Crucible aec5136b-4422-434a-a943-db33c736240e has session id: bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103102392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Upstairs opts: Upstairs UUID: aec5136b-4422-434a-a943-db33c736240e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103130779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Crucible stats registered with UUID: aec5136b-4422-434a-a943-db33c736240e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103148537Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:55.103157725Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103283429Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103311536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103324704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103469596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103481484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"aec5136b-4422-434a-a943-db33c736240e active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103497372Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103506131Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"0","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.10351564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"1","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103525279Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"2","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103534848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103544646Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103554325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62165)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103621987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59732)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.1036699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:47730)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.103700907Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104291652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"0","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104316169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"1","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104365662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"2","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104627629Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[0]R flush_numbers[0..12]: [182, 182, 177, 179, 182, 172, 182, 182, 179, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104653896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[0]R generation[0..12]: [107, 107, 104, 104, 107, 102, 107, 107, 104, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104665114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104676023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[1]R flush_numbers[0..12]: [182, 182, 177, 179, 182, 172, 182, 182, 179, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104686562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[1]R generation[0..12]: [107, 107, 104, 104, 107, 102, 107, 107, 104, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.10469705Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104708299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[2]R flush_numbers[0..12]: [182, 182, 177, 179, 182, 172, 182, 182, 179, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104718757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[2]R generation[0..12]: [107, 107, 104, 104, 107, 102, 107, 107, 104, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104729286Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104740115Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104749484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104759192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104768551Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.10477971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"":"downstairs","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104789498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104798157Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"0","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104807916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"1","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104817825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"client":"2","":"downstairs","session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104827794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104836992Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"aec5136b-4422-434a-a943-db33c736240e is now active with session: bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104846791Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.10485539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"session_id":"bd7594a2-b647-4d1e-8fcc-18bdd3acc4bb"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104864389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Activated sub_volume aec5136b-4422-434a-a943-db33c736240e","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104872268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.104880287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.105138434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"task":"crutest"} 01/30 Flush 02/30 Write block 393 len 8192 data: 10 9 03/30 Flush 04/30 Write block 273 len 20480 data: 6 5 8 9 10 05/30 Read block 422 len 28672 06/30 Read block 209 len 8192 07/30 Read block 390 len 36864 08/30 Read block 488 len 36864 09/30 Write block 209 len 28672 data: 14 13 13 14 14 14 11 10/30 Write block 452 len 24576 data: 8 11 12 9 10 11 11/30 Write block 219 len 12288 data: 10 12 13 12/30 Flush 13/30 Write block 350 len 36864 data: 8 8 11 8 7 7 6 8 8 14/30 Read block 483 len 28672 15/30 Write block 295 len 16384 data: 10 9 9 7 16/30 Write block 181 len 4096 data: 10 17/30 Read block 270 len 20480 18/30 Write block 231 len 4096 data: 6 19/30 Read block 147 len 8192 20/30 Flush 21/30 Write block 356 len 8192 data: 7 9 22/30 Write block 223 len 24576 data: 9 9 8 7 7 5 23/30 Flush 24/30 Write block 146 len 16384 data: 12 11 13 13 25/30 Read block 426 len 24576 26/30 Read block 479 len 36864 27/30 Read block 131 len 20480 28/30 Write block 134 len 24576 data: 15 12 11 9 10 11 29/30 Write block 548 len 24576 data: 6 7 8 7 7 7 30/30 Write block 407 len 24576 data: 7 6 5 4 5 8 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 6 Sent Done Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 29 34 Done 26 27 0 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:57:55.113271033Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:57:55.113460659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1621,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 182 182 182 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 104 104 104 179 179 179 F F F 4 080-099 107 107 107 182 182 182 F F F 5 100-119 102 102 102 172 172 172 F F F 6 120-139 107 107 107 182 182 182 T T F <--- 7 140-159 107 107 107 182 182 182 T T F <--- 8 160-179 104 104 104 179 179 179 F F F 9 180-199 108 108 107 186 186 182 F F F <--- 10 200-219 108 108 107 185 185 182 F F F <--- 11 220-239 108 108 107 187 187 182 F F F <--- 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 101 185 185 171 F F F <--- 14 280-299 108 108 107 186 186 182 F F F <--- 15 300-319 105 105 105 180 180 180 F F F 16 320-339 102 102 102 172 172 172 F F F 17 340-359 108 108 102 187 187 172 F F F <--- 18 360-379 106 106 106 181 181 181 F F F 19 380-399 108 108 106 184 184 181 F F F <--- 20 400-419 106 106 106 181 181 181 T T F <--- 21 420-439 102 102 102 172 172 172 F F F 22 440-459 108 108 104 185 185 178 F F F <--- 23 460-479 101 101 101 171 171 171 F F F 24 480-499 107 107 107 182 182 182 F F F 25 500-519 107 107 107 182 182 182 F F F 26 520-539 104 104 104 179 179 179 F F F 27 540-559 107 107 107 182 182 182 T T F <--- 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 108, Max flush: 187 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 182 182 182 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 104 104 104 179 179 179 F F F 4 080-099 107 107 107 182 182 182 F F F 5 100-119 102 102 102 172 172 172 F F F 6 120-139 109 109 109 188 188 188 F F F 7 140-159 109 109 109 188 188 188 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 108 108 108 186 186 186 F F F 10 200-219 108 108 108 185 185 185 F F F 11 220-239 108 108 108 187 187 187 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 108 185 185 185 F F F 14 280-299 108 108 108 186 186 186 F F F 15 300-319 105 105 105 180 180 180 F F F 16 320-339 102 102 102 172 172 172 F F F 17 340-359 108 108 108 187 187 187 F F F 18 360-379 106 106 106 181 181 181 F F F 19 380-399 108 108 108 184 184 184 F F F 20 400-419 109 109 109 188 188 188 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 101 101 101 171 171 171 F F F 24 480-499 107 107 107 182 182 182 F F F 25 500-519 107 107 107 182 182 182 F F F 26 520-539 104 104 104 179 179 179 F F F 27 540-559 109 109 109 188 188 188 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 109, Max flush: 188 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.367410806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:09.374507186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.374566638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.381456155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.381718391Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.381754977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.381764116Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.381775824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Crucible 117fd92b-e6d5-4d0c-8e41-0c1031e70dec has session id: 3338b248-898e-40da-b6dd-bcc335724fde","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382158906Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Upstairs opts: Upstairs UUID: 117fd92b-e6d5-4d0c-8e41-0c1031e70dec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382186462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Crucible stats registered with UUID: 117fd92b-e6d5-4d0c-8e41-0c1031e70dec","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.38219995Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:09.382209169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382342972Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.38235951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382369389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382524669Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382541147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"117fd92b-e6d5-4d0c-8e41-0c1031e70dec active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382564494Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382578672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"0","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382590761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"1","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.3826004Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"2","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382610228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382620077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382629876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"ds_connection connected from Ok(127.0.0.1:63881)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382683519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"ds_connection connected from Ok(127.0.0.1:59454)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382694098Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"ds_connection connected from Ok(127.0.0.1:48439)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.382704176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383353174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"2","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383379311Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"1","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383573896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"0","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383796638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[0]R flush_numbers[0..12]: [182, 182, 177, 179, 182, 172, 188, 188, 179, 186, 185, 187]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383822874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[0]R generation[0..12]: [107, 107, 104, 104, 107, 102, 109, 109, 104, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383834203Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383845082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[1]R flush_numbers[0..12]: [182, 182, 177, 179, 182, 172, 188, 188, 179, 186, 185, 187]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.38385822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[1]R generation[0..12]: [107, 107, 104, 104, 107, 102, 109, 109, 104, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383868929Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383880007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[2]R flush_numbers[0..12]: [182, 182, 177, 179, 182, 172, 188, 188, 179, 186, 185, 187]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383890446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[2]R generation[0..12]: [107, 107, 104, 104, 107, 102, 109, 109, 104, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383900865Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383911663Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383921022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383930691Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.38393989Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383949228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"":"downstairs","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383958807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383967426Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"0","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383976975Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"1","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383986474Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"client":"2","":"downstairs","session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.383997492Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.384006631Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"117fd92b-e6d5-4d0c-8e41-0c1031e70dec is now active with session: 3338b248-898e-40da-b6dd-bcc335724fde","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.38401649Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.384025139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"session_id":"3338b248-898e-40da-b6dd-bcc335724fde"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.384034078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Activated sub_volume 117fd92b-e6d5-4d0c-8e41-0c1031e70dec","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.384042037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.384050086Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.384295055Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} 01/30 Write block 303 len 12288 data: 9 8 8 02/30 Read block 508 len 28672 03/30 Read block 291 len 8192 04/30 Write block 326 len 32768 data: 6 8 8 9 10 5 7 8 05/30 Write block 516 len 28672 data: 14 12 9 11 13 12 8 06/30 Read block 492 len 28672 07/30 Read block 327 len 4096 08/30 Read block 406 len 12288 09/30 Read block 360 len 32768 10/30 Read block 438 len 36864 11/30 Write block 345 len 36864 data: 11 11 12 10 10 9 9 12 9 12/30 Write block 72 len 40960 data: 10 9 12 15 14 14 13 14 11 8 13/30 Write block 356 len 4096 data: 8 14/30 Read block 206 len 24576 15/30 Read block 269 len 24576 16/30 Write block 10 len 24576 data: 6 9 11 8 9 9 17/30 Write block 298 len 36864 data: 8 6 5 5 6 10 9 9 9 18/30 Read block 527 len 4096 19/30 Flush 20/30 Write block 361 len 36864 data: 8 9 10 10 10 10 9 7 8 21/30 Write block 208 len 12288 data: 16 15 14 22/30 Write block 472 len 40960 data: 6 5 6 7 6 7 6 8 9 8 23/30 Write block 402 len 4096 data: 10 24/30 Read block 232 len 28672 25/30 Read block 258 len 8192 26/30 Write block 388 len 20480 data: 6 8 9 9 10 27/30 Read block 111 len 28672 28/30 Write block 281 len 24576 data: 12 11 13 10 11 12 29/30 Write block 99 len 24576 data: 8 9 10 9 8 7 30/30 Write block 122 len 20480 data: 10 12 13 14 14 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 3 3 35 Done 1 27 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:09.392408716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:09.392576864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1630,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 110 110 182 189 189 F F F <--- 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 104 110 110 179 189 189 F F F <--- 4 080-099 107 110 110 182 189 189 F T T <--- 5 100-119 102 102 102 172 172 172 F T T <--- 6 120-139 109 109 109 188 188 188 F T T <--- 7 140-159 109 109 109 188 188 188 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 108 108 108 186 186 186 F F F 10 200-219 108 108 108 185 185 185 F T T <--- 11 220-239 108 108 108 187 187 187 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 108 185 185 185 F F F 14 280-299 108 110 110 186 189 189 F T T <--- 15 300-319 105 110 110 180 189 189 T F F <--- 16 320-339 102 110 110 172 189 189 F F F <--- 17 340-359 108 110 110 187 189 189 F F F <--- 18 360-379 106 106 106 181 181 181 F T T <--- 19 380-399 108 108 108 184 184 184 F T T <--- 20 400-419 109 109 109 188 188 188 F T T <--- 21 420-439 102 102 102 172 172 172 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 101 101 101 171 171 171 F T T <--- 24 480-499 107 107 107 182 182 182 F T T <--- 25 500-519 107 110 110 182 189 189 F F F <--- 26 520-539 104 110 110 179 189 189 F F F <--- 27 540-559 109 109 109 188 188 188 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 110, Max flush: 189 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 110 110 110 189 189 189 F F F 4 080-099 111 111 111 190 190 190 F F F 5 100-119 111 111 111 190 190 190 F F F 6 120-139 111 111 111 190 190 190 F F F 7 140-159 109 109 109 188 188 188 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 108 108 108 186 186 186 F F F 10 200-219 111 111 111 190 190 190 F F F 11 220-239 108 108 108 187 187 187 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 108 185 185 185 F F F 14 280-299 111 111 111 190 190 190 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 110 110 110 189 189 189 F F F 18 360-379 111 111 111 190 190 190 F F F 19 380-399 111 111 111 190 190 190 F F F 20 400-419 111 111 111 190 190 190 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 111 111 111 190 190 190 F F F 24 480-499 111 111 111 190 190 190 F F F 25 500-519 110 110 110 189 189 189 F F F 26 520-539 110 110 110 189 189 189 F F F 27 540-559 109 109 109 188 188 188 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 111, Max flush: 190 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.660437867Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:23.66743225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.667487343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.674156157Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.674416354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.674441071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.6744494Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.674461179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Crucible f0d5e934-6a8b-4545-a1b9-ad1719d4dfe5 has session id: 690c10ad-8add-43f0-8f48-bbc57334bd77","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67484257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Upstairs opts: Upstairs UUID: f0d5e934-6a8b-4545-a1b9-ad1719d4dfe5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.674873946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Crucible stats registered with UUID: f0d5e934-6a8b-4545-a1b9-ad1719d4dfe5","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.674885185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:23.674894394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675045195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675076061Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675086759Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675223872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67524067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"f0d5e934-6a8b-4545-a1b9-ad1719d4dfe5 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675274206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675283494Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"0","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675293233Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"1","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675302912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"2","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675312381Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67532229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675334238Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"ds_connection connected from Ok(127.0.0.1:62965)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675375583Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"ds_connection connected from Ok(127.0.0.1:47381)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67540122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"ds_connection connected from Ok(127.0.0.1:34651)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.675520634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67611067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"1","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676128307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"2","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676139046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"0","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67650294Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[0]R flush_numbers[0..12]: [189, 182, 177, 189, 190, 190, 190, 188, 179, 186, 190, 187]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676528077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[0]R generation[0..12]: [110, 107, 104, 110, 111, 111, 111, 109, 104, 108, 111, 108]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676539155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676549924Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[1]R flush_numbers[0..12]: [189, 182, 177, 189, 190, 190, 190, 188, 179, 186, 190, 187]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676560383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[1]R generation[0..12]: [110, 107, 104, 110, 111, 111, 111, 109, 104, 108, 111, 108]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676570741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.6765816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[2]R flush_numbers[0..12]: [189, 182, 177, 189, 190, 190, 190, 188, 179, 186, 190, 187]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676594198Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[2]R generation[0..12]: [110, 107, 104, 110, 111, 111, 111, 109, 104, 108, 111, 108]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676604597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676615346Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676624614Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676634173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676643282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676652551Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"":"downstairs","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676666969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676675548Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"0","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676684927Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"1","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676694456Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"client":"2","":"downstairs","session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676704134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676713133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"f0d5e934-6a8b-4545-a1b9-ad1719d4dfe5 is now active with session: 690c10ad-8add-43f0-8f48-bbc57334bd77","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676722852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676731501Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"session_id":"690c10ad-8add-43f0-8f48-bbc57334bd77"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.67674176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Activated sub_volume f0d5e934-6a8b-4545-a1b9-ad1719d4dfe5","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676749659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.676757657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.677063559Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} 01/30 Write block 218 len 20480 data: 11 11 13 14 11 02/30 Read block 72 len 24576 03/30 Read block 94 len 28672 04/30 Read block 539 len 32768 05/30 Write block 370 len 40960 data: 8 6 6 5 6 8 9 9 9 10 06/30 Write block 561 len 12288 data: 4 4 6 07/30 Read block 519 len 32768 08/30 Read block 407 len 16384 09/30 Write block 97 len 28672 data: 8 9 9 10 11 10 9 10/30 Read block 32 len 32768 11/30 Write block 566 len 16384 data: 6 6 6 6 12/30 Read block 528 len 12288 13/30 Write block 300 len 12288 data: 6 6 7 14/30 Write block 10 len 12288 data: 7 10 12 15/30 Write block 382 len 24576 data: 7 7 8 7 6 7 16/30 Read block 268 len 4096 17/30 Write block 301 len 4096 data: 7 18/30 Read block 450 len 36864 19/30 Write block 545 len 36864 data: 8 6 6 7 8 9 8 8 8 20/30 Flush 21/30 Flush 22/30 Write block 11 len 28672 data: 11 13 9 10 10 8 9 23/30 Read block 129 len 4096 24/30 Flush 25/30 Write block 510 len 8192 data: 8 9 26/30 Read block 123 len 12288 27/30 Read block 121 len 32768 28/30 Read block 566 len 20480 29/30 Write block 342 len 32768 data: 9 10 8 12 12 13 11 11 30/30 Read block 413 len 16384 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:23.685993926Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:23.686168014Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1639,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 112 112 189 193 193 F F F <--- 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 110 110 110 189 189 189 F F F 4 080-099 111 112 112 190 191 191 F F F <--- 5 100-119 111 112 112 190 191 191 F F F <--- 6 120-139 111 111 111 190 190 190 F F F 7 140-159 109 109 109 188 188 188 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 108 108 108 186 186 186 F F F 10 200-219 111 112 112 190 191 191 F F F <--- 11 220-239 108 112 112 187 191 191 F F F <--- 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 108 185 185 185 F F F 14 280-299 111 111 111 190 190 190 F F F 15 300-319 110 112 112 189 191 191 F F F <--- 16 320-339 110 110 110 189 189 189 F F F 17 340-359 110 110 110 189 189 189 F T T <--- 18 360-379 111 112 112 190 191 191 F F F <--- 19 380-399 111 112 112 190 191 191 F F F <--- 20 400-419 111 111 111 190 190 190 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 111 111 111 190 190 190 F F F 24 480-499 111 111 111 190 190 190 F F F 25 500-519 110 110 110 189 189 189 F T T <--- 26 520-539 110 110 110 189 189 189 F F F 27 540-559 109 112 112 188 191 191 F F F <--- 28 560-579 96 112 112 164 191 191 F F F <--- 29 580-599 98 98 98 167 167 167 F F F Max gen: 112, Max flush: 193 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 193 193 193 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 110 110 110 189 189 189 F F F 4 080-099 112 112 112 191 191 191 F F F 5 100-119 112 112 112 191 191 191 F F F 6 120-139 111 111 111 190 190 190 F F F 7 140-159 109 109 109 188 188 188 F F F 8 160-179 104 104 104 179 179 179 F F F 9 180-199 108 108 108 186 186 186 F F F 10 200-219 112 112 112 191 191 191 F F F 11 220-239 112 112 112 191 191 191 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 108 185 185 185 F F F 14 280-299 111 111 111 190 190 190 F F F 15 300-319 112 112 112 191 191 191 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 113 113 113 194 194 194 F F F 18 360-379 112 112 112 191 191 191 F F F 19 380-399 112 112 112 191 191 191 F F F 20 400-419 111 111 111 190 190 190 F F F 21 420-439 102 102 102 172 172 172 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 111 111 111 190 190 190 F F F 24 480-499 111 111 111 190 190 190 F F F 25 500-519 113 113 113 194 194 194 F F F 26 520-539 110 110 110 189 189 189 F F F 27 540-559 112 112 112 191 191 191 F F F 28 560-579 112 112 112 191 191 191 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 113, Max flush: 194 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.954855231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.962936167Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963253816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963280553Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963289332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96330074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Crucible 2f2508b4-f1a2-401d-ab3c-b33b84370ead has session id: ccb416c2-47ed-4924-996e-70101f220479","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963689671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Upstairs opts: Upstairs UUID: 2f2508b4-f1a2-401d-ab3c-b33b84370ead, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963728236Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Crucible stats registered with UUID: 2f2508b4-f1a2-401d-ab3c-b33b84370ead","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963742244Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:37.963751383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963907833Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96393714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.963948108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96409251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964105058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"2f2508b4-f1a2-401d-ab3c-b33b84370ead active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964119806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964128655Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"0","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964138064Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"1","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964147543Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"2","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964157262Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96416703Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964176789Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"ds_connection connected from Ok(127.0.0.1:38524)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964282476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"ds_connection connected from Ok(127.0.0.1:43572)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964299174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"ds_connection connected from Ok(127.0.0.1:44830)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.964392142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965020752Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"0","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96503652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"2","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965047919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"1","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965404643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[0]R flush_numbers[0..12]: [193, 182, 177, 189, 191, 191, 190, 188, 179, 186, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965439379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[0]R generation[0..12]: [112, 107, 104, 110, 112, 112, 111, 109, 104, 108, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965450408Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965461296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[1]R flush_numbers[0..12]: [193, 182, 177, 189, 191, 191, 190, 188, 179, 186, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965471685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[1]R generation[0..12]: [112, 107, 104, 110, 112, 112, 111, 109, 104, 108, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965482214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965493012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[2]R flush_numbers[0..12]: [193, 182, 177, 189, 191, 191, 190, 188, 179, 186, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965503531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[2]R generation[0..12]: [112, 107, 104, 110, 112, 112, 111, 109, 104, 108, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96551392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965524698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965534117Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965543796Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965554714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965564043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"":"downstairs","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965573742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965582431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"0","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.96559198Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"1","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965601518Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"client":"2","":"downstairs","session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965611517Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965620516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"2f2508b4-f1a2-401d-ab3c-b33b84370ead is now active with session: ccb416c2-47ed-4924-996e-70101f220479","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965630335Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965638914Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"session_id":"ccb416c2-47ed-4924-996e-70101f220479"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965648063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Activated sub_volume 2f2508b4-f1a2-401d-ab3c-b33b84370ead","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965656022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965664011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.965947495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"task":"crutest"} 01/30 Write block 193 len 4096 data: 14 02/30 Read block 177 len 8192 03/30 Read block 314 len 36864 04/30 Write block 122 len 12288 data: 11 13 14 05/30 Write block 344 len 20480 data: 9 13 13 14 12 06/30 Read block 323 len 4096 07/30 Write block 74 len 16384 data: 13 16 15 15 08/30 Read block 325 len 4096 09/30 Read block 567 len 32768 10/30 Read block 216 len 28672 11/30 Write block 93 len 8192 data: 6 5 12/30 Write block 204 len 24576 data: 11 14 15 16 17 16 13/30 Write block 175 len 28672 data: 13 12 11 7 9 11 11 14/30 Flush 15/30 Write block 279 len 20480 data: 9 11 13 12 14 16/30 Write block 219 len 8192 data: 12 14 17/30 Write block 505 len 32768 data: 10 9 9 11 8 9 10 10 18/30 Write block 493 len 12288 data: 8 6 4 19/30 Write block 308 len 4096 data: 10 20/30 Read block 264 len 32768 21/30 Write block 466 len 16384 data: 9 7 7 5 22/30 Write block 152 len 20480 data: 12 13 13 11 12 23/30 Write block 129 len 4096 data: 14 24/30 Read block 544 len 28672 25/30 Write block 431 len 24576 data: 7 7 7 8 8 8 26/30 Read block 135 len 24576 27/30 Read block 561 len 24576 28/30 Read block 398 len 32768 29/30 Write block 522 len 16384 data: 9 8 7 10 30/30 Read block 488 len 32768 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 0 25 Done 30 5 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:37.973730038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:37.973917414Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1648,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 193 193 193 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 114 110 114 195 189 195 F F F <--- 4 080-099 114 112 114 195 191 195 F F F <--- 5 100-119 112 112 112 191 191 191 F F F 6 120-139 114 111 114 195 190 195 T T T <--- 7 140-159 109 109 109 188 188 188 T F T <--- 8 160-179 114 104 114 195 179 195 F F F <--- 9 180-199 114 108 114 195 186 195 F T F <--- 10 200-219 114 112 114 195 191 195 T F T <--- 11 220-239 112 112 112 191 191 191 T F T <--- 12 240-259 101 101 101 171 171 171 F F F 13 260-279 108 108 108 185 185 185 T F T <--- 14 280-299 111 111 111 190 190 190 T F T <--- 15 300-319 112 112 112 191 191 191 T F T <--- 16 320-339 110 110 110 189 189 189 F F F 17 340-359 114 113 114 195 194 195 F T F <--- 18 360-379 112 112 112 191 191 191 F F F 19 380-399 112 112 112 191 191 191 F F F 20 400-419 111 111 111 190 190 190 F F F 21 420-439 102 102 102 172 172 172 T F T <--- 22 440-459 108 108 108 185 185 185 F F F 23 460-479 111 111 111 190 190 190 T F T <--- 24 480-499 111 111 111 190 190 190 T F T <--- 25 500-519 113 113 113 194 194 194 T F T <--- 26 520-539 110 110 110 189 189 189 T F T <--- 27 540-559 112 112 112 191 191 191 F F F 28 560-579 112 112 112 191 191 191 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 114, Max flush: 195 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 193 193 193 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 114 114 114 195 195 195 F F F 4 080-099 114 114 114 195 195 195 F F F 5 100-119 112 112 112 191 191 191 F F F 6 120-139 115 115 115 196 196 196 F F F 7 140-159 115 115 115 196 196 196 F F F 8 160-179 114 114 114 195 195 195 F F F 9 180-199 114 114 114 195 195 195 F F F 10 200-219 115 115 115 196 196 196 F F F 11 220-239 115 115 115 196 196 196 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 115 115 115 196 196 196 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 114 114 114 195 195 195 F F F 18 360-379 112 112 112 191 191 191 F F F 19 380-399 112 112 112 191 191 191 F F F 20 400-419 111 111 111 190 190 190 F F F 21 420-439 115 115 115 196 196 196 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 115 115 115 196 196 196 F F F 24 480-499 115 115 115 196 196 196 F F F 25 500-519 115 115 115 196 196 196 F F F 26 520-539 115 115 115 196 196 196 F F F 27 540-559 112 112 112 191 191 191 F F F 28 560-579 112 112 112 191 191 191 F F F 29 580-599 98 98 98 167 167 167 F F F Max gen: 115, Max flush: 196 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.24490541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.25287283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253181581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253208567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253217006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253228515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Crucible 831eee7a-87f1-4f3e-b602-d990ff51d230 has session id: cd63c65f-157e-4060-9fa1-5e30212dff17","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253565442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Upstairs opts: Upstairs UUID: 831eee7a-87f1-4f3e-b602-d990ff51d230, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253598808Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Crucible stats registered with UUID: 831eee7a-87f1-4f3e-b602-d990ff51d230","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253609596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:52.253618675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253778175Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253795843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253805791Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253912388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253926466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"831eee7a-87f1-4f3e-b602-d990ff51d230 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253950063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253959322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"0","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.25397163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"1","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253981509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"2","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.253991118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254013035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254022894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"ds_connection connected from Ok(127.0.0.1:38552)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.25405669Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"ds_connection connected from Ok(127.0.0.1:62096)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254081566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"ds_connection connected from Ok(127.0.0.1:58634)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254093155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254786417Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"0","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254825872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"1","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.254837361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"2","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255131023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[0]R flush_numbers[0..12]: [193, 182, 177, 195, 195, 191, 196, 196, 195, 195, 196, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.25515755Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[0]R generation[0..12]: [112, 107, 104, 114, 114, 112, 115, 115, 114, 114, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255168809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255179717Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[1]R flush_numbers[0..12]: [193, 182, 177, 195, 195, 191, 196, 196, 195, 195, 196, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255192326Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[1]R generation[0..12]: [112, 107, 104, 114, 114, 112, 115, 115, 114, 114, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255202844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255213713Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[2]R flush_numbers[0..12]: [193, 182, 177, 195, 195, 191, 196, 196, 195, 195, 196, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255224062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[2]R generation[0..12]: [112, 107, 104, 114, 114, 112, 115, 115, 114, 114, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.25523444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255245249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255254458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255264216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255273525Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255282834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"":"downstairs","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255292613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255301182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"0","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255310671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"1","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255320129Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"client":"2","":"downstairs","session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255331228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255340227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"831eee7a-87f1-4f3e-b602-d990ff51d230 is now active with session: cd63c65f-157e-4060-9fa1-5e30212dff17","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255349936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255358644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"session_id":"cd63c65f-157e-4060-9fa1-5e30212dff17"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255367553Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Activated sub_volume 831eee7a-87f1-4f3e-b602-d990ff51d230","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255375422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255383431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.255692272Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"task":"crutest"} 01/30 Write block 65 len 36864 data: 7 6 7 5 6 6 11 11 10 02/30 Write block 422 len 40960 data: 13 14 14 15 12 11 9 9 6 8 03/30 Read block 547 len 36864 04/30 Read block 498 len 20480 05/30 Write block 388 len 8192 data: 7 9 06/30 Flush 07/30 Write block 555 len 4096 data: 7 08/30 Write block 484 len 36864 data: 10 10 7 7 5 3 3 4 6 09/30 Write block 72 len 36864 data: 12 11 14 17 16 16 14 15 12 10/30 Write block 7 len 8192 data: 3 4 11/30 Flush 12/30 Write block 129 len 4096 data: 15 13/30 Write block 503 len 40960 data: 8 9 11 10 10 12 9 10 11 11 14/30 Flush 15/30 Write block 307 len 40960 data: 10 11 8 8 8 7 5 7 8 6 16/30 Write block 590 len 36864 data: 8 9 7 6 6 6 6 4 4 17/30 Write block 540 len 8192 data: 16 13 18/30 Write block 365 len 12288 data: 11 11 10 19/30 Read block 468 len 4096 20/30 Read block 28 len 12288 21/30 Read block 259 len 16384 22/30 Read block 470 len 32768 23/30 Write block 495 len 8192 data: 5 5 24/30 Write block 182 len 28672 data: 11 15 11 8 9 9 8 25/30 Write block 114 len 4096 data: 7 26/30 Read block 469 len 36864 27/30 Read block 279 len 36864 28/30 Write block 100 len 24576 data: 11 12 11 10 8 7 29/30 Write block 103 len 36864 data: 11 9 8 7 10 12 12 11 10 30/30 Write block 83 len 36864 data: 6 10 12 13 12 17 14 12 9 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:58:52.263486424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:58:52.263659372Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1657,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 112 116 198 193 198 F F F <--- 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 114 116 198 195 198 F F F <--- 4 080-099 116 114 116 198 195 198 T F T <--- 5 100-119 112 112 112 191 191 191 T F T <--- 6 120-139 116 115 116 199 196 199 F F F <--- 7 140-159 115 115 115 196 196 196 F F F 8 160-179 114 114 114 195 195 195 F F F 9 180-199 114 114 114 195 195 195 T F T <--- 10 200-219 115 115 115 196 196 196 F F F 11 220-239 115 115 115 196 196 196 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 115 115 115 196 196 196 T F T <--- 16 320-339 110 110 110 189 189 189 F F F 17 340-359 114 114 114 195 195 195 F F F 18 360-379 112 112 112 191 191 191 T F T <--- 19 380-399 116 112 116 197 191 197 F F F <--- 20 400-419 111 111 111 190 190 190 F F F 21 420-439 116 115 116 197 196 197 F F F <--- 22 440-459 108 108 108 185 185 185 F F F 23 460-479 115 115 115 196 196 196 F F F 24 480-499 116 115 116 198 196 198 T F T <--- 25 500-519 116 115 116 199 196 199 F F F <--- 26 520-539 115 115 115 196 196 196 F F F 27 540-559 116 112 116 198 191 198 T F T <--- 28 560-579 112 112 112 191 191 191 F F F 29 580-599 98 98 98 167 167 167 T F T <--- Max gen: 116, Max flush: 199 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 198 198 198 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 117 117 117 200 200 200 F F F 5 100-119 117 117 117 200 200 200 F F F 6 120-139 116 116 116 199 199 199 F F F 7 140-159 115 115 115 196 196 196 F F F 8 160-179 114 114 114 195 195 195 F F F 9 180-199 117 117 117 200 200 200 F F F 10 200-219 115 115 115 196 196 196 F F F 11 220-239 115 115 115 196 196 196 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 117 117 117 200 200 200 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 114 114 114 195 195 195 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 111 111 111 190 190 190 F F F 21 420-439 116 116 116 197 197 197 F F F 22 440-459 108 108 108 185 185 185 F F F 23 460-479 115 115 115 196 196 196 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 116 116 116 199 199 199 F F F 26 520-539 115 115 115 196 196 196 F F F 27 540-559 117 117 117 200 200 200 F F F 28 560-579 112 112 112 191 191 191 F F F 29 580-599 117 117 117 200 200 200 F F F Max gen: 117, Max flush: 200 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.525752446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:06.532832418Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.532887981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.539748221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540025606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540052813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540061361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.54007572Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Crucible 78b0fa66-879c-464b-8c22-c87899d4eb82 has session id: 1707fd58-a16c-46a2-8348-c6c5e4dcd742","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540405558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Upstairs opts: Upstairs UUID: 78b0fa66-879c-464b-8c22-c87899d4eb82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540482838Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Crucible stats registered with UUID: 78b0fa66-879c-464b-8c22-c87899d4eb82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540494617Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:06.540504095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540654176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540685272Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540696011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540823685Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540840283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"78b0fa66-879c-464b-8c22-c87899d4eb82 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540853531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.54086242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"0","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540872289Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"1","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540881977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"2","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540891576Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540915743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540928382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"ds_connection connected from Ok(127.0.0.1:49418)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540960717Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"ds_connection connected from Ok(127.0.0.1:37582)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540971806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"ds_connection connected from Ok(127.0.0.1:60206)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.540982005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.541687985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"0","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.54172556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"1","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.541755817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"2","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542036241Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[0]R flush_numbers[0..12]: [198, 182, 177, 198, 200, 200, 199, 196, 195, 200, 196, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542055869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[0]R generation[0..12]: [116, 107, 104, 116, 117, 117, 116, 115, 114, 117, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542066607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542077536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[1]R flush_numbers[0..12]: [198, 182, 177, 198, 200, 200, 199, 196, 195, 200, 196, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542088074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[1]R generation[0..12]: [116, 107, 104, 116, 117, 117, 116, 115, 114, 117, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542098583Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542111661Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[2]R flush_numbers[0..12]: [198, 182, 177, 198, 200, 200, 199, 196, 195, 200, 196, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.54212218Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[2]R generation[0..12]: [116, 107, 104, 116, 117, 117, 116, 115, 114, 117, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542132629Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542143327Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542152626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542162385Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542171684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542180893Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"":"downstairs","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542190731Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.54219944Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"0","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542209019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"1","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542218518Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"client":"2","":"downstairs","session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542228217Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542237126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"78b0fa66-879c-464b-8c22-c87899d4eb82 is now active with session: 1707fd58-a16c-46a2-8348-c6c5e4dcd742","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542246794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542255493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"session_id":"1707fd58-a16c-46a2-8348-c6c5e4dcd742"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542265922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Activated sub_volume 78b0fa66-879c-464b-8c22-c87899d4eb82","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542273791Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.54228183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.542598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} 01/30 Write block 184 len 24576 data: 12 9 10 10 9 10 02/30 Write block 353 len 8192 data: 10 8 03/30 Read block 413 len 20480 04/30 Write block 103 len 36864 data: 12 10 9 8 11 13 13 12 11 05/30 Write block 185 len 16384 data: 10 11 11 10 06/30 Read block 440 len 36864 07/30 Flush 08/30 Flush 09/30 Write block 525 len 4096 data: 11 10/30 Read block 366 len 40960 11/30 Flush 12/30 Write block 408 len 8192 data: 7 6 13/30 Write block 152 len 4096 data: 13 14/30 Write block 534 len 16384 data: 11 11 12 10 15/30 Write block 209 len 16384 data: 17 15 14 15 16/30 Read block 98 len 16384 17/30 Read block 225 len 4096 18/30 Read block 186 len 20480 19/30 Write block 591 len 20480 data: 10 8 7 7 7 20/30 Write block 15 len 16384 data: 11 9 10 8 21/30 Flush 22/30 Write block 171 len 28672 data: 14 15 14 13 14 13 12 23/30 Read block 319 len 16384 24/30 Write block 446 len 16384 data: 10 10 8 8 25/30 Write block 344 len 20480 data: 10 14 14 15 13 26/30 Read block 398 len 32768 27/30 Read block 336 len 12288 28/30 Read block 363 len 32768 29/30 Write block 342 len 28672 data: 10 11 11 15 15 16 14 30/30 Read block 20 len 36864 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 3 30 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:06.551042769Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:06.551228845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1666,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 118 118 198 204 204 F F F <--- 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 117 117 117 200 200 200 F F F 5 100-119 117 118 118 200 201 201 F F F <--- 6 120-139 116 116 116 199 199 199 F F F 7 140-159 115 118 118 196 204 204 F F F <--- 8 160-179 114 114 114 195 195 195 F T T <--- 9 180-199 117 118 118 200 201 201 T F F <--- 10 200-219 115 118 118 196 204 204 F F F <--- 11 220-239 115 115 115 196 196 196 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 117 117 117 200 200 200 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 114 118 118 195 201 201 T T T <--- 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 111 118 118 190 204 204 F F F <--- 21 420-439 116 116 116 197 197 197 F F F 22 440-459 108 108 108 185 185 185 F T T <--- 23 460-479 115 115 115 196 196 196 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 116 116 116 199 199 199 F F F 26 520-539 115 118 118 196 204 204 F F F <--- 27 540-559 117 117 117 200 200 200 F F F 28 560-579 112 112 112 191 191 191 F F F 29 580-599 117 118 118 200 204 204 F F F <--- Max gen: 118, Max flush: 204 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 204 204 204 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 117 117 117 200 200 200 F F F 5 100-119 118 118 118 201 201 201 F F F 6 120-139 116 116 116 199 199 199 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 118 118 118 201 201 201 F F F 10 200-219 118 118 118 204 204 204 F F F 11 220-239 115 115 115 196 196 196 F F F 12 240-259 101 101 101 171 171 171 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 117 117 117 200 200 200 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 118 118 118 204 204 204 F F F 21 420-439 116 116 116 197 197 197 F F F 22 440-459 119 119 119 205 205 205 F F F 23 460-479 115 115 115 196 196 196 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 116 116 116 199 199 199 F F F 26 520-539 118 118 118 204 204 204 F F F 27 540-559 117 117 117 200 200 200 F F F 28 560-579 112 112 112 191 191 191 F F F 29 580-599 118 118 118 204 204 204 F F F Max gen: 119, Max flush: 205 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.8180501Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.825969105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826254979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826279216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826287705Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826299073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Crucible 4197de00-922d-4a70-9d4e-c50f60406acf has session id: 864e14f6-433f-4a0c-a016-bf2575e0cc62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826661337Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Upstairs opts: Upstairs UUID: 4197de00-922d-4a70-9d4e-c50f60406acf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826690244Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Crucible stats registered with UUID: 4197de00-922d-4a70-9d4e-c50f60406acf","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826701152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:20.826710281Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826850533Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.826868451Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.82687846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827045909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827069696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"4197de00-922d-4a70-9d4e-c50f60406acf active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827085984Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827094842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"0","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827104481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"1","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.82711419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"2","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827123809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827133728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827143606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"ds_connection connected from Ok(127.0.0.1:53686)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827205178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"ds_connection connected from Ok(127.0.0.1:47351)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827230795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"ds_connection connected from Ok(127.0.0.1:34980)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827303836Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827886632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"0","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.8279052Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"1","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.827933106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"2","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828237288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[0]R flush_numbers[0..12]: [204, 182, 177, 198, 200, 201, 199, 204, 205, 201, 204, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828267654Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[0]R generation[0..12]: [118, 107, 104, 116, 117, 118, 116, 118, 119, 118, 118, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828278802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828289701Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[1]R flush_numbers[0..12]: [204, 182, 177, 198, 200, 201, 199, 204, 205, 201, 204, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.82830017Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[1]R generation[0..12]: [118, 107, 104, 116, 117, 118, 116, 118, 119, 118, 118, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828310788Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828321637Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[2]R flush_numbers[0..12]: [204, 182, 177, 198, 200, 201, 199, 204, 205, 201, 204, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828332066Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[2]R generation[0..12]: [118, 107, 104, 116, 117, 118, 116, 118, 119, 118, 118, 115]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828342564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828353273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828362732Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.82837239Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828383349Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828392678Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"":"downstairs","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828402767Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828411455Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"0","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828421474Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"1","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828431273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"client":"2","":"downstairs","session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828441302Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828450431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"4197de00-922d-4a70-9d4e-c50f60406acf is now active with session: 864e14f6-433f-4a0c-a016-bf2575e0cc62","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828460329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828468988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"session_id":"864e14f6-433f-4a0c-a016-bf2575e0cc62"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828477987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Activated sub_volume 4197de00-922d-4a70-9d4e-c50f60406acf","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828486056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828494145Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.828809355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"task":"crutest"} 01/30 Write block 568 len 16384 data: 7 7 4 3 02/30 Read block 84 len 8192 03/30 Read block 364 len 24576 04/30 Read block 12 len 40960 05/30 Write block 254 len 16384 data: 8 8 9 6 06/30 Read block 566 len 12288 07/30 Write block 552 len 32768 data: 9 9 5 8 5 6 7 6 08/30 Read block 275 len 24576 09/30 Flush 10/30 Write block 304 len 36864 data: 10 10 10 11 12 9 9 9 8 11/30 Write block 236 len 20480 data: 10 9 9 7 8 12/30 Read block 408 len 12288 13/30 Write block 445 len 8192 data: 10 11 14/30 Read block 499 len 32768 15/30 Write block 466 len 24576 data: 10 8 8 6 5 5 16/30 Read block 142 len 36864 17/30 Read block 483 len 24576 18/30 Write block 101 len 8192 data: 13 12 19/30 Read block 494 len 40960 20/30 Read block 176 len 40960 21/30 Read block 246 len 12288 22/30 Read block 46 len 20480 23/30 Write block 535 len 28672 data: 12 13 11 12 16 17 14 24/30 Read block 123 len 4096 25/30 Write block 205 len 36864 data: 15 16 17 18 18 16 15 16 15 26/30 Write block 300 len 20480 data: 7 8 8 11 11 27/30 Read block 185 len 20480 28/30 Read block 586 len 36864 29/30 Read block 465 len 4096 30/30 Write block 120 len 36864 data: 7 7 12 14 15 15 15 14 12 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 10 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:20.837710636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:20.837896383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1675,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 204 204 204 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 117 117 117 200 200 200 F F F 5 100-119 118 118 118 201 201 201 T T F <--- 6 120-139 116 116 116 199 199 199 T T F <--- 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 118 118 118 201 201 201 F F F 10 200-219 118 118 118 204 204 204 T T F <--- 11 220-239 115 115 115 196 196 196 T T F <--- 12 240-259 120 120 101 206 206 171 T T F <--- 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 117 117 117 200 200 200 T T F <--- 16 320-339 110 110 110 189 189 189 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 118 118 118 204 204 204 F F F 21 420-439 116 116 116 197 197 197 F F F 22 440-459 119 119 119 205 205 205 T T F <--- 23 460-479 115 115 115 196 196 196 T T F <--- 24 480-499 117 117 117 200 200 200 F F F 25 500-519 116 116 116 199 199 199 F F F 26 520-539 118 118 118 204 204 204 T T F <--- 27 540-559 120 120 117 206 206 200 T T F <--- 28 560-579 120 120 112 206 206 191 F F T <--- 29 580-599 118 118 118 204 204 204 F F F Max gen: 120, Max flush: 206 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 204 204 204 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 117 117 117 200 200 200 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 118 118 118 201 201 201 F F F 10 200-219 121 121 121 207 207 207 F F F 11 220-239 121 121 121 207 207 207 F F F 12 240-259 121 121 121 207 207 207 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 121 121 121 207 207 207 F F F 16 320-339 110 110 110 189 189 189 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 118 118 118 204 204 204 F F F 21 420-439 116 116 116 197 197 197 F F F 22 440-459 121 121 121 207 207 207 F F F 23 460-479 121 121 121 207 207 207 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 116 116 116 199 199 199 F F F 26 520-539 121 121 121 207 207 207 F F F 27 540-559 121 121 121 207 207 207 F F F 28 560-579 120 120 120 206 206 206 F F F 29 580-599 118 118 118 204 204 204 F F F Max gen: 121, Max flush: 207 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.090802612Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:35.097830221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.097885044Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.104565697Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.104828283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.10485309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.104861539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.104872898Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Crucible 8e26a1b7-b43c-428e-8fa6-13f752942f42 has session id: 8611051d-0ae6-4e94-868a-76a197486e89","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.10524634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Upstairs opts: Upstairs UUID: 8e26a1b7-b43c-428e-8fa6-13f752942f42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105288235Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Crucible stats registered with UUID: 8e26a1b7-b43c-428e-8fa6-13f752942f42","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105302623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:35.105312022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105465362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105497148Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105507997Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105636891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105663337Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"8e26a1b7-b43c-428e-8fa6-13f752942f42 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105673546Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105682225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"0","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105691804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"1","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105701183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"2","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105710561Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.10572052Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105730539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"ds_connection connected from Ok(127.0.0.1:61409)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105834466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"ds_connection connected from Ok(127.0.0.1:38935)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.105859442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"ds_connection connected from Ok(127.0.0.1:61474)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.10595677Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106496502Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"1","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106530457Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"0","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106541396Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"2","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106940395Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[0]R flush_numbers[0..12]: [204, 182, 177, 198, 200, 207, 207, 204, 205, 201, 207, 207]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106957293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[0]R generation[0..12]: [118, 107, 104, 116, 117, 121, 121, 118, 119, 118, 121, 121]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106970562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.1069816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[1]R flush_numbers[0..12]: [204, 182, 177, 198, 200, 207, 207, 204, 205, 201, 207, 207]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.106992469Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[1]R generation[0..12]: [118, 107, 104, 116, 117, 121, 121, 118, 119, 118, 121, 121]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107002987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107013976Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[2]R flush_numbers[0..12]: [204, 182, 177, 198, 200, 207, 207, 204, 205, 201, 207, 207]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107024445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[2]R generation[0..12]: [118, 107, 104, 116, 117, 121, 121, 118, 119, 118, 121, 121]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107034863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107045692Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107055181Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.10706489Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107074268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107083437Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"":"downstairs","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107093296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107101915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"0","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107113223Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"1","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107127642Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"client":"2","":"downstairs","session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.10713731Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107146359Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"8e26a1b7-b43c-428e-8fa6-13f752942f42 is now active with session: 8611051d-0ae6-4e94-868a-76a197486e89","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107156108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107164737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"session_id":"8611051d-0ae6-4e94-868a-76a197486e89"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107173726Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Activated sub_volume 8e26a1b7-b43c-428e-8fa6-13f752942f42","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107181705Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107189814Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.107510283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} 01/30 Write block 127 len 32768 data: 15 13 16 14 18 20 15 16 02/30 Write block 441 len 36864 data: 7 7 8 9 11 12 11 9 9 03/30 Flush 04/30 Write block 236 len 12288 data: 11 10 10 05/30 Write block 574 len 8192 data: 7 9 06/30 Read block 126 len 32768 07/30 Write block 593 len 4096 data: 8 08/30 Write block 429 len 4096 data: 10 09/30 Flush 10/30 Write block 200 len 4096 data: 11 11/30 Flush 12/30 Write block 320 len 40960 data: 8 7 7 7 6 8 7 9 9 10 13/30 Read block 228 len 20480 14/30 Write block 0 len 20480 data: 2 2 3 3 2 15/30 Flush 16/30 Write block 505 len 4096 data: 12 17/30 Write block 569 len 12288 data: 8 5 4 18/30 Write block 92 len 24576 data: 6 7 6 7 9 9 19/30 Read block 10 len 32768 20/30 Write block 186 len 24576 data: 12 12 11 11 8 8 21/30 Write block 530 len 12288 data: 9 10 11 22/30 Flush 23/30 Write block 567 len 12288 data: 7 8 9 24/30 Read block 48 len 32768 25/30 Write block 405 len 36864 data: 8 7 8 8 7 5 6 9 9 26/30 Write block 184 len 36864 data: 13 11 13 13 12 12 9 9 12 27/30 Read block 198 len 32768 28/30 Write block 514 len 28672 data: 10 12 15 13 10 12 14 29/30 Read block 320 len 12288 30/30 Read block 420 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:35.11628864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:35.116466427Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1684,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 122 122 204 211 211 F F F <--- 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 117 122 122 200 212 212 F F F <--- 5 100-119 121 121 121 207 207 207 F F F 6 120-139 121 122 122 207 208 208 T F F <--- 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 118 122 122 201 212 212 F T T <--- 10 200-219 121 122 122 207 210 210 F F F <--- 11 220-239 121 122 122 207 209 209 F F F <--- 12 240-259 121 121 121 207 207 207 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 121 121 121 207 207 207 F F F 16 320-339 110 122 122 189 211 211 F F F <--- 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 118 118 118 204 204 204 F T T <--- 21 420-439 116 122 122 197 209 209 F F F <--- 22 440-459 121 122 122 207 208 208 F F F <--- 23 460-479 121 121 121 207 207 207 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 116 122 122 199 212 212 F T T <--- 26 520-539 121 122 122 207 212 212 F T T <--- 27 540-559 121 121 121 207 207 207 F F F 28 560-579 120 122 122 206 212 212 F T T <--- 29 580-599 118 122 122 204 209 209 F F F <--- Max gen: 122, Max flush: 212 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 104 104 104 177 177 177 F F F 3 060-079 116 116 116 198 198 198 F F F 4 080-099 122 122 122 212 212 212 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 123 123 123 213 213 213 F F F 10 200-219 122 122 122 210 210 210 F F F 11 220-239 122 122 122 209 209 209 F F F 12 240-259 121 121 121 207 207 207 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 121 121 121 207 207 207 F F F 16 320-339 122 122 122 211 211 211 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 116 116 116 197 197 197 F F F 20 400-419 123 123 123 213 213 213 F F F 21 420-439 122 122 122 209 209 209 F F F 22 440-459 122 122 122 208 208 208 F F F 23 460-479 121 121 121 207 207 207 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 123 123 123 213 213 213 F F F 26 520-539 123 123 123 213 213 213 F F F 27 540-559 121 121 121 207 207 207 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 122 122 122 209 209 209 F F F Max gen: 123, Max flush: 213 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.373727675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.381739219Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382027402Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382051409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382059788Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382071247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Crucible 7a3b3b7e-d77f-4a60-b180-baf93aa97c9d has session id: db6b8647-29f5-4aa1-a66d-6412b9929e7d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382491153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Upstairs opts: Upstairs UUID: 7a3b3b7e-d77f-4a60-b180-baf93aa97c9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.38252052Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Crucible stats registered with UUID: 7a3b3b7e-d77f-4a60-b180-baf93aa97c9d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382531688Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:49.382541027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382682169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382713105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382724184Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382871755Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382895522Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"7a3b3b7e-d77f-4a60-b180-baf93aa97c9d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382906871Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.38291563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"0","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382925458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"1","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382935007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"2","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382944756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.382954625Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.383035834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38018)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.383060191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54579)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.383097097Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47027)","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.383107745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.383745464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"0","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.38377592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"1","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.383809816Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"2","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384114288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[0]R flush_numbers[0..12]: [211, 182, 177, 198, 212, 207, 208, 204, 205, 213, 210, 209]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384142564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[0]R generation[0..12]: [122, 107, 104, 116, 122, 121, 122, 118, 119, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384153743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384164541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[1]R flush_numbers[0..12]: [211, 182, 177, 198, 212, 207, 208, 204, 205, 213, 210, 209]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.38417507Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[1]R generation[0..12]: [122, 107, 104, 116, 122, 121, 122, 118, 119, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384185489Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384196437Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[2]R flush_numbers[0..12]: [211, 182, 177, 198, 212, 207, 208, 204, 205, 213, 210, 209]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384208996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[2]R generation[0..12]: [122, 107, 104, 116, 122, 121, 122, 118, 119, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384219444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384230273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384239582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.38424913Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384258449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384267618Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"":"downstairs","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384277407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384286106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"0","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384295775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"1","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384305223Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"client":"2","":"downstairs","session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384314982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384324181Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"7a3b3b7e-d77f-4a60-b180-baf93aa97c9d is now active with session: db6b8647-29f5-4aa1-a66d-6412b9929e7d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.38433398Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384342679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"session_id":"db6b8647-29f5-4aa1-a66d-6412b9929e7d"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384352947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Activated sub_volume 7a3b3b7e-d77f-4a60-b180-baf93aa97c9d","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384360876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384368915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.384674447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"task":"crutest"} 01/30 Write block 95 len 20480 data: 8 10 10 10 10 02/30 Write block 57 len 32768 data: 14 13 12 11 9 8 8 7 03/30 Read block 106 len 36864 04/30 Flush 05/30 Write block 390 len 12288 data: 10 10 11 06/30 Write block 228 len 16384 data: 6 5 6 7 07/30 Read block 84 len 12288 08/30 Write block 390 len 4096 data: 11 09/30 Write block 487 len 28672 data: 8 6 4 4 5 7 9 10/30 Read block 428 len 20480 11/30 Write block 449 len 20480 data: 10 7 8 9 12 12/30 Read block 241 len 12288 13/30 Read block 345 len 4096 14/30 Read block 237 len 8192 15/30 Read block 114 len 32768 16/30 Write block 453 len 12288 data: 13 13 10 17/30 Write block 88 len 16384 data: 18 15 13 10 18/30 Read block 375 len 4096 19/30 Flush 20/30 Write block 300 len 4096 data: 8 21/30 Read block 462 len 36864 22/30 Read block 468 len 20480 23/30 Read block 344 len 12288 24/30 Read block 119 len 12288 25/30 Write block 461 len 32768 data: 10 12 16 12 13 11 9 9 26/30 Read block 542 len 20480 27/30 Read block 178 len 40960 28/30 Read block 557 len 12288 29/30 Read block 89 len 40960 30/30 Read block 449 len 8192 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 3 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 1 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 23 23 Done 26 26 3 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T00:59:49.392962585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T00:59:49.393141773Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1693,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 124 124 124 214 214 214 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 124 124 124 215 215 214 F F F <--- 5 100-119 121 121 121 207 207 207 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 123 123 123 213 213 213 F F F 10 200-219 122 122 122 210 210 210 F F F 11 220-239 124 124 122 215 215 209 F F T <--- 12 240-259 121 121 121 207 207 207 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 121 121 121 207 207 207 T T F <--- 16 320-339 122 122 122 211 211 211 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 124 124 116 215 215 197 F F T <--- 20 400-419 123 123 123 213 213 213 F F F 21 420-439 122 122 122 209 209 209 F F F 22 440-459 124 124 122 215 215 208 F F F <--- 23 460-479 121 121 121 207 207 207 T T F <--- 24 480-499 124 124 117 215 215 200 F F F <--- 25 500-519 123 123 123 213 213 213 F F F 26 520-539 123 123 123 213 213 213 F F F 27 540-559 121 121 121 207 207 207 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 122 122 122 209 209 209 F F F Max gen: 124, Max flush: 215 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 124 124 124 214 214 214 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 124 124 124 215 215 215 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 123 123 123 213 213 213 F F F 10 200-219 122 122 122 210 210 210 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 121 121 121 207 207 207 F F F 13 260-279 115 115 115 196 196 196 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 125 125 125 216 216 216 F F F 16 320-339 122 122 122 211 211 211 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 124 124 124 215 215 215 F F F 20 400-419 123 123 123 213 213 213 F F F 21 420-439 122 122 122 209 209 209 F F F 22 440-459 124 124 124 215 215 215 F F F 23 460-479 125 125 125 216 216 216 F F F 24 480-499 124 124 124 215 215 215 F F F 25 500-519 123 123 123 213 213 213 F F F 26 520-539 123 123 123 213 213 213 F F F 27 540-559 121 121 121 207 207 207 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 122 122 122 209 209 209 F F F Max gen: 125, Max flush: 216 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.638975529Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647099069Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647412449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647455744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.64748293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647495979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Crucible a08676bd-b683-475f-bf80-8f7b804aab3d has session id: 1343bf4a-62ef-49b2-b123-95c2d8abc3fb","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647786542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Upstairs opts: Upstairs UUID: a08676bd-b683-475f-bf80-8f7b804aab3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647830306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Crucible stats registered with UUID: a08676bd-b683-475f-bf80-8f7b804aab3d","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647848114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:03.647857323Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.647988036Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648005384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648017592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648170503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648183821Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"a08676bd-b683-475f-bf80-8f7b804aab3d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648200249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648209118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"0","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648218627Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"1","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648228006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"2","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648237545Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648247263Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648257112Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:32974)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.64834935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:50842)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648374257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46965)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.648385756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649045432Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"2","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649074008Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"1","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649124852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"0","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649405316Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[0]R flush_numbers[0..12]: [211, 182, 214, 214, 215, 207, 208, 204, 205, 213, 210, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649439872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[0]R generation[0..12]: [122, 107, 124, 124, 124, 121, 122, 118, 119, 123, 122, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649451031Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649461859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[1]R flush_numbers[0..12]: [211, 182, 214, 214, 215, 207, 208, 204, 205, 213, 210, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649472488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[1]R generation[0..12]: [122, 107, 124, 124, 124, 121, 122, 118, 119, 123, 122, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649482887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649493565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[2]R flush_numbers[0..12]: [211, 182, 214, 214, 215, 207, 208, 204, 205, 213, 210, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649503994Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[2]R generation[0..12]: [122, 107, 124, 124, 124, 121, 122, 118, 119, 123, 122, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649514343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649525161Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.64953446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649544329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649553598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649562846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"":"downstairs","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649573825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649582554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"0","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649592153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"1","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649601652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"client":"2","":"downstairs","session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.64961133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649620199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"a08676bd-b683-475f-bf80-8f7b804aab3d is now active with session: 1343bf4a-62ef-49b2-b123-95c2d8abc3fb","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649630108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649638797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"session_id":"1343bf4a-62ef-49b2-b123-95c2d8abc3fb"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649647806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Activated sub_volume a08676bd-b683-475f-bf80-8f7b804aab3d","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649655735Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649663854Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.649992872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"task":"crutest"} 01/30 Flush 02/30 Write block 522 len 28672 data: 10 9 8 12 9 8 7 03/30 Write block 315 len 36864 data: 9 7 6 6 6 9 8 8 8 04/30 Read block 194 len 36864 05/30 Read block 267 len 32768 06/30 Write block 453 len 16384 data: 14 14 11 11 07/30 Read block 452 len 32768 08/30 Write block 538 len 40960 data: 13 17 18 15 12 9 10 9 7 7 09/30 Write block 155 len 12288 data: 12 13 11 10/30 Read block 37 len 28672 11/30 Read block 214 len 12288 12/30 Read block 223 len 24576 13/30 Write block 257 len 40960 data: 7 8 8 7 6 7 7 9 7 7 14/30 Read block 536 len 28672 15/30 Read block 214 len 8192 16/30 Write block 399 len 40960 data: 4 7 9 11 11 11 9 8 9 9 17/30 Read block 363 len 32768 18/30 Read block 171 len 36864 19/30 Read block 309 len 28672 20/30 Read block 196 len 28672 21/30 Read block 499 len 32768 22/30 Read block 395 len 36864 23/30 Read block 86 len 4096 24/30 Read block 3 len 24576 25/30 Write block 501 len 32768 data: 5 7 9 10 13 11 11 13 26/30 Flush 27/30 Write block 491 len 12288 data: 6 8 10 28/30 Write block 390 len 4096 data: 12 29/30 Read block 200 len 24576 30/30 Read block 131 len 28672 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 28 0 29 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1000)) Some(JobId(1025)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:03.659823635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:03.6600196Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1702,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 124 124 124 214 214 214 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 124 124 124 215 215 215 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 126 118 126 218 204 218 F F F <--- 8 160-179 119 119 119 205 205 205 F F F 9 180-199 123 123 123 213 213 213 F F F 10 200-219 122 122 122 210 210 210 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 126 121 126 218 207 218 F F F <--- 13 260-279 126 115 126 218 196 218 F F F <--- 14 280-299 115 115 115 196 196 196 F F F 15 300-319 126 125 126 218 216 218 F F F <--- 16 320-339 126 122 126 218 211 218 F F F <--- 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 126 124 126 218 215 218 T F T <--- 20 400-419 126 123 126 218 213 218 F F F <--- 21 420-439 122 122 122 209 209 209 F F F 22 440-459 126 124 126 218 215 218 F F F <--- 23 460-479 125 125 125 216 216 216 F F F 24 480-499 124 124 124 215 215 215 T F T <--- 25 500-519 126 123 126 218 213 218 F F F <--- 26 520-539 126 123 126 218 213 218 F F F <--- 27 540-559 126 121 126 218 207 218 F F F <--- 28 560-579 123 123 123 213 213 213 F F F 29 580-599 122 122 122 209 209 209 F F F Max gen: 126, Max flush: 218 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 124 124 124 214 214 214 F F F 3 060-079 124 124 124 214 214 214 F F F 4 080-099 124 124 124 215 215 215 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 122 122 122 208 208 208 F F F 7 140-159 126 126 126 218 218 218 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 123 123 123 213 213 213 F F F 10 200-219 122 122 122 210 210 210 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 126 126 126 218 218 218 F F F 13 260-279 126 126 126 218 218 218 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 218 218 218 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 117 117 117 200 200 200 F F F 19 380-399 127 127 127 219 219 219 F F F 20 400-419 126 126 126 218 218 218 F F F 21 420-439 122 122 122 209 209 209 F F F 22 440-459 126 126 126 218 218 218 F F F 23 460-479 125 125 125 216 216 216 F F F 24 480-499 127 127 127 219 219 219 F F F 25 500-519 126 126 126 218 218 218 F F F 26 520-539 126 126 126 218 218 218 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 122 122 122 209 209 209 F F F Max gen: 127, Max flush: 219 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.914409721Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.922358973Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.922644707Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.922673693Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.922684472Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.92269597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Crucible b065a5ec-b7fd-4d6f-82b7-abb2e7095d6e has session id: 58abb2e7-51cb-4081-a633-8b8f342caf93","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923046136Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Upstairs opts: Upstairs UUID: b065a5ec-b7fd-4d6f-82b7-abb2e7095d6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923074702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Crucible stats registered with UUID: b065a5ec-b7fd-4d6f-82b7-abb2e7095d6e","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923085521Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:17.9230947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923243701Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923274677Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923285585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923412959Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923432687Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"b065a5ec-b7fd-4d6f-82b7-abb2e7095d6e active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923450695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923459523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"0","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923469662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"1","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923479361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"2","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923491309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923501338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923516636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"ds_connection connected from Ok(127.0.0.1:45550)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923551392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"ds_connection connected from Ok(127.0.0.1:55360)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923623393Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"ds_connection connected from Ok(127.0.0.1:38247)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.923635051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924282169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"2","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924337892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"1","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924391845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"0","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924683988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[0]R flush_numbers[0..12]: [211, 182, 214, 214, 215, 207, 208, 218, 205, 213, 210, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924709025Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[0]R generation[0..12]: [122, 107, 124, 124, 124, 121, 122, 126, 119, 123, 122, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924719964Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924730942Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[1]R flush_numbers[0..12]: [211, 182, 214, 214, 215, 207, 208, 218, 205, 213, 210, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924741581Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[1]R generation[0..12]: [122, 107, 124, 124, 124, 121, 122, 126, 119, 123, 122, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924755229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924766228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[2]R flush_numbers[0..12]: [211, 182, 214, 214, 215, 207, 208, 218, 205, 213, 210, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924776746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[2]R generation[0..12]: [122, 107, 124, 124, 124, 121, 122, 126, 119, 123, 122, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924787285Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924798124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924807422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924817151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.92482634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924835849Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"":"downstairs","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924845728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924854357Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"0","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924863925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"1","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924873384Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"client":"2","":"downstairs","session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924883123Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924892142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"b065a5ec-b7fd-4d6f-82b7-abb2e7095d6e is now active with session: 58abb2e7-51cb-4081-a633-8b8f342caf93","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.92490357Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924912209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"session_id":"58abb2e7-51cb-4081-a633-8b8f342caf93"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924921218Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Activated sub_volume b065a5ec-b7fd-4d6f-82b7-abb2e7095d6e","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924929137Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.924937246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.92522013Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"task":"crutest"} 01/30 Write block 207 len 28672 data: 18 19 19 17 16 17 16 02/30 Read block 587 len 24576 03/30 Read block 477 len 12288 04/30 Read block 18 len 4096 05/30 Write block 522 len 32768 data: 11 10 9 13 10 9 8 7 06/30 Write block 185 len 24576 data: 12 14 14 13 13 10 07/30 Read block 561 len 36864 08/30 Read block 51 len 36864 09/30 Read block 119 len 12288 10/30 Write block 431 len 12288 data: 9 8 8 11/30 Write block 141 len 40960 data: 12 12 10 11 9 13 12 14 14 11 12/30 Write block 135 len 4096 data: 13 13/30 Read block 227 len 20480 14/30 Write block 375 len 20480 data: 9 10 10 10 11 15/30 Read block 401 len 24576 16/30 Read block 496 len 28672 17/30 Read block 398 len 12288 18/30 Write block 60 len 8192 data: 12 10 19/30 Write block 142 len 36864 data: 13 11 12 10 14 13 15 15 12 20/30 Write block 581 len 12288 data: 11 11 11 21/30 Flush 22/30 Write block 529 len 4096 data: 8 23/30 Read block 506 len 40960 24/30 Read block 68 len 8192 25/30 Flush 26/30 Flush 27/30 Write block 388 len 20480 data: 8 10 13 11 12 28/30 Read block 501 len 8192 29/30 Read block 426 len 8192 30/30 Read block 135 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:17.933621925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:17.933794403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1711,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 124 124 124 214 214 214 F F F 3 060-079 128 124 128 220 214 220 F F F <--- 4 080-099 124 124 124 215 215 215 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 128 122 128 220 208 220 F F F <--- 7 140-159 128 126 128 220 218 220 F F F <--- 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 123 128 220 213 220 F F F <--- 10 200-219 128 122 128 220 210 220 F F F <--- 11 220-239 124 124 124 215 215 215 F F F 12 240-259 126 126 126 218 218 218 F F F 13 260-279 126 126 126 218 218 218 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 218 218 218 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 128 117 128 220 200 220 F F F <--- 19 380-399 127 127 127 219 219 219 T F T <--- 20 400-419 126 126 126 218 218 218 F F F 21 420-439 128 122 128 220 209 220 F F F <--- 22 440-459 126 126 126 218 218 218 F F F 23 460-479 125 125 125 216 216 216 F F F 24 480-499 127 127 127 219 219 219 F F F 25 500-519 126 126 126 218 218 218 F F F 26 520-539 128 126 128 221 218 221 F F F <--- 27 540-559 126 126 126 218 218 218 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 128 122 128 220 209 220 F F F <--- Max gen: 128, Max flush: 221 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 F F F 2 040-059 124 124 124 214 214 214 F F F 3 060-079 128 128 128 220 220 220 F F F 4 080-099 124 124 124 215 215 215 F F F 5 100-119 121 121 121 207 207 207 F F F 6 120-139 128 128 128 220 220 220 F F F 7 140-159 128 128 128 220 220 220 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 128 128 128 220 220 220 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 126 126 126 218 218 218 F F F 13 260-279 126 126 126 218 218 218 F F F 14 280-299 115 115 115 196 196 196 F F F 15 300-319 126 126 126 218 218 218 F F F 16 320-339 126 126 126 218 218 218 F F F 17 340-359 119 119 119 205 205 205 F F F 18 360-379 128 128 128 220 220 220 F F F 19 380-399 129 129 129 222 222 222 F F F 20 400-419 126 126 126 218 218 218 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 126 126 126 218 218 218 F F F 23 460-479 125 125 125 216 216 216 F F F 24 480-499 127 127 127 219 219 219 F F F 25 500-519 126 126 126 218 218 218 F F F 26 520-539 128 128 128 221 221 221 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 128 128 128 220 220 220 F F F Max gen: 129, Max flush: 222 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.176736786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.184713634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185031864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185055741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185064299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185075688Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Crucible 9ff546c9-9a2d-48a5-a7ad-63d622b9f8e3 has session id: f6157da8-0624-407f-b1da-1908b230f804","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185475907Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Upstairs opts: Upstairs UUID: 9ff546c9-9a2d-48a5-a7ad-63d622b9f8e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185492165Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Crucible stats registered with UUID: 9ff546c9-9a2d-48a5-a7ad-63d622b9f8e3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185505214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:32.185516002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185662274Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185678442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.18568834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.18584523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185862708Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"9ff546c9-9a2d-48a5-a7ad-63d622b9f8e3 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185885765Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185894954Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"0","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185904533Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"1","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185914072Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"2","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.18592362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185933479Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.185943318Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"ds_connection connected from Ok(127.0.0.1:55660)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.186029837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"ds_connection connected from Ok(127.0.0.1:34001)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.18608832Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"ds_connection connected from Ok(127.0.0.1:41088)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.186107587Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.186680295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"1","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.186698122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"0","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.186708911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"2","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187047908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[0]R flush_numbers[0..12]: [211, 182, 214, 220, 215, 207, 220, 220, 205, 220, 220, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187073365Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[0]R generation[0..12]: [122, 107, 124, 128, 124, 121, 128, 128, 119, 128, 128, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187084313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187095302Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[1]R flush_numbers[0..12]: [211, 182, 214, 220, 215, 207, 220, 220, 205, 220, 220, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187105771Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[1]R generation[0..12]: [122, 107, 124, 128, 124, 121, 128, 128, 119, 128, 128, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187116129Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187126978Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[2]R flush_numbers[0..12]: [211, 182, 214, 220, 215, 207, 220, 220, 205, 220, 220, 215]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187137477Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[2]R generation[0..12]: [122, 107, 124, 128, 124, 121, 128, 128, 119, 128, 128, 124]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187148085Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187158804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187169672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187179431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.18718861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187198019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"":"downstairs","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187207568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187216287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"0","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187225885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"1","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187235544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"client":"2","":"downstairs","session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187245163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187254042Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"9ff546c9-9a2d-48a5-a7ad-63d622b9f8e3 is now active with session: f6157da8-0624-407f-b1da-1908b230f804","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187263851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187272419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"session_id":"f6157da8-0624-407f-b1da-1908b230f804"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187281478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Activated sub_volume 9ff546c9-9a2d-48a5-a7ad-63d622b9f8e3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187289457Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187297456Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.187615556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"task":"crutest"} 01/30 Write block 265 len 8192 data: 8 8 02/30 Write block 212 len 40960 data: 18 17 15 12 11 10 12 13 15 15 03/30 Read block 440 len 32768 04/30 Read block 73 len 20480 05/30 Write block 118 len 40960 data: 8 7 8 8 13 15 16 16 16 16 06/30 Flush 07/30 Write block 352 len 12288 data: 13 11 9 08/30 Write block 139 len 28672 data: 12 8 13 14 12 13 11 09/30 Write block 409 len 12288 data: 8 6 7 10/30 Write block 486 len 16384 data: 8 9 7 5 11/30 Flush 12/30 Write block 286 len 4096 data: 13 13/30 Write block 272 len 24576 data: 6 7 6 9 10 11 14/30 Write block 441 len 36864 data: 8 8 9 10 12 13 12 10 11 15/30 Write block 258 len 28672 data: 9 9 8 7 8 8 10 16/30 Read block 52 len 16384 17/30 Write block 460 len 4096 data: 11 18/30 Read block 442 len 28672 19/30 Read block 290 len 16384 20/30 Read block 45 len 32768 21/30 Write block 36 len 28672 data: 6 6 5 5 8 8 9 22/30 Read block 570 len 20480 23/30 Write block 532 len 32768 data: 12 12 12 13 14 12 14 18 24/30 Read block 288 len 40960 25/30 Write block 337 len 12288 data: 11 10 8 26/30 Read block 471 len 4096 27/30 Write block 313 len 24576 data: 6 8 10 8 7 7 28/30 Read block 228 len 28672 29/30 Read block 86 len 36864 30/30 Read block 340 len 36864 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:32.196262339Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:32.196431848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1720,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 107 107 107 182 182 182 T T F <--- 2 040-059 124 124 124 214 214 214 T T F <--- 3 060-079 128 128 128 220 220 220 F F F 4 080-099 124 124 124 215 215 215 F F F 5 100-119 130 130 121 223 223 207 F F F <--- 6 120-139 130 130 128 224 224 220 F F F <--- 7 140-159 130 130 128 224 224 220 F F F <--- 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 130 130 128 223 223 220 F F F <--- 11 220-239 130 130 124 223 223 215 F F F <--- 12 240-259 126 126 126 218 218 218 T T F <--- 13 260-279 130 130 126 223 223 218 T T F <--- 14 280-299 115 115 115 196 196 196 T T F <--- 15 300-319 126 126 126 218 218 218 T T F <--- 16 320-339 126 126 126 218 218 218 T T F <--- 17 340-359 130 130 119 224 224 205 F F F <--- 18 360-379 128 128 128 220 220 220 F F F 19 380-399 129 129 129 222 222 222 F F F 20 400-419 130 130 126 224 224 218 F F F <--- 21 420-439 128 128 128 220 220 220 F F F 22 440-459 126 126 126 218 218 218 T T F <--- 23 460-479 125 125 125 216 216 216 T T F <--- 24 480-499 130 130 127 224 224 219 F F F <--- 25 500-519 126 126 126 218 218 218 F F F 26 520-539 128 128 128 221 221 221 T T F <--- 27 540-559 126 126 126 218 218 218 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 128 128 128 220 220 220 F F F Max gen: 130, Max flush: 224 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 131 131 131 225 225 225 F F F 2 040-059 131 131 131 225 225 225 F F F 3 060-079 128 128 128 220 220 220 F F F 4 080-099 124 124 124 215 215 215 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 130 130 130 224 224 224 F F F 7 140-159 130 130 130 224 224 224 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 130 130 130 223 223 223 F F F 12 240-259 131 131 131 225 225 225 F F F 13 260-279 131 131 131 225 225 225 F F F 14 280-299 131 131 131 225 225 225 F F F 15 300-319 131 131 131 225 225 225 F F F 16 320-339 131 131 131 225 225 225 F F F 17 340-359 130 130 130 224 224 224 F F F 18 360-379 128 128 128 220 220 220 F F F 19 380-399 129 129 129 222 222 222 F F F 20 400-419 130 130 130 224 224 224 F F F 21 420-439 128 128 128 220 220 220 F F F 22 440-459 131 131 131 225 225 225 F F F 23 460-479 131 131 131 225 225 225 F F F 24 480-499 130 130 130 224 224 224 F F F 25 500-519 126 126 126 218 218 218 F F F 26 520-539 131 131 131 225 225 225 F F F 27 540-559 126 126 126 218 218 218 F F F 28 560-579 123 123 123 213 213 213 F F F 29 580-599 128 128 128 220 220 220 F F F Max gen: 131, Max flush: 225 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.460890542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.468787221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469095332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469123818Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469132347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469143686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Crucible afde34aa-b3ad-4f1b-8911-65ce82993d37 has session id: 1f00494a-afd3-4f43-87bb-d13e3813aa8f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469466315Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Upstairs opts: Upstairs UUID: afde34aa-b3ad-4f1b-8911-65ce82993d37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469487722Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Crucible stats registered with UUID: afde34aa-b3ad-4f1b-8911-65ce82993d37","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.46950018Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:46.469509279Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469645672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.46966186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469671889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469813771Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469837598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"afde34aa-b3ad-4f1b-8911-65ce82993d37 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469851246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469859975Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"0","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469869513Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"1","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469879032Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"2","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469888491Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.46989823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469908109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39211)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.469994748Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42352)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.470129461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56466)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.470140519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.470672902Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"0","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.470711207Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"1","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.470726005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"2","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471032506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[0]R flush_numbers[0..12]: [211, 225, 225, 220, 215, 223, 224, 224, 205, 220, 223, 223]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471060512Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[0]R generation[0..12]: [122, 131, 131, 128, 124, 130, 130, 130, 119, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471073891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471084839Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[1]R flush_numbers[0..12]: [211, 225, 225, 220, 215, 223, 224, 224, 205, 220, 223, 223]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471095288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[1]R generation[0..12]: [122, 131, 131, 128, 124, 130, 130, 130, 119, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471105657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471116485Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[2]R flush_numbers[0..12]: [211, 225, 225, 220, 215, 223, 224, 224, 205, 220, 223, 223]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471126894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[2]R generation[0..12]: [122, 131, 131, 128, 124, 130, 130, 130, 119, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471137303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471148161Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.47115755Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471167349Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471176618Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471186047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"":"downstairs","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471195985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471204874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"0","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471215973Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"1","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471225681Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"client":"2","":"downstairs","session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.47123558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471244809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"afde34aa-b3ad-4f1b-8911-65ce82993d37 is now active with session: 1f00494a-afd3-4f43-87bb-d13e3813aa8f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471254698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471263297Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"session_id":"1f00494a-afd3-4f43-87bb-d13e3813aa8f"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471272536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Activated sub_volume afde34aa-b3ad-4f1b-8911-65ce82993d37","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471284634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471292893Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.471519394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"task":"crutest"} 01/30 Flush 02/30 Write block 27 len 20480 data: 8 8 8 9 9 03/30 Write block 75 len 32768 data: 18 17 17 15 16 13 9 7 04/30 Read block 85 len 20480 05/30 Read block 133 len 12288 06/30 Flush 07/30 Write block 346 len 12288 data: 16 17 15 08/30 Read block 283 len 16384 09/30 Write block 66 len 28672 data: 7 8 6 7 7 12 13 10/30 Write block 575 len 28672 data: 10 11 10 9 9 10 12 11/30 Read block 405 len 24576 12/30 Write block 426 len 36864 data: 13 12 10 11 7 10 9 9 9 13/30 Read block 249 len 4096 14/30 Write block 248 len 20480 data: 14 13 7 10 11 15/30 Write block 541 len 28672 data: 16 13 10 11 10 8 8 16/30 Read block 12 len 40960 17/30 Write block 533 len 8192 data: 13 13 18/30 Read block 433 len 12288 19/30 Write block 229 len 12288 data: 6 7 8 20/30 Write block 306 len 12288 data: 11 12 13 21/30 Read block 590 len 12288 22/30 Read block 465 len 8192 23/30 Read block 126 len 32768 24/30 Write block 126 len 4096 data: 17 25/30 Write block 498 len 32768 data: 5 6 5 6 8 10 11 14 26/30 Write block 374 len 20480 data: 7 10 11 11 11 27/30 Read block 416 len 32768 28/30 Read block 514 len 36864 29/30 Read block 164 len 32768 30/30 Write block 523 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 13 1 15 Done 23 11 23 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:00:46.479653643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:00:46.479819002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1729,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 131 131 131 225 225 225 F F F 3 060-079 132 132 132 227 227 227 T T T 4 080-099 132 132 132 227 227 227 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 130 130 130 224 224 224 T F T <--- 7 140-159 130 130 130 224 224 224 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 130 130 130 223 223 223 T F T <--- 12 240-259 131 131 131 225 225 225 T T T 13 260-279 131 131 131 225 225 225 F F F 14 280-299 131 131 131 225 225 225 F F F 15 300-319 131 131 131 225 225 225 T F T <--- 16 320-339 131 131 131 225 225 225 F F F 17 340-359 130 130 130 224 224 224 T T T 18 360-379 128 128 128 220 220 220 T F T <--- 19 380-399 129 129 129 222 222 222 F F F 20 400-419 130 130 130 224 224 224 F F F 21 420-439 128 128 128 220 220 220 T T T 22 440-459 131 131 131 225 225 225 F F F 23 460-479 131 131 131 225 225 225 F F F 24 480-499 130 130 130 224 224 224 T F T <--- 25 500-519 126 126 126 218 218 218 T F T <--- 26 520-539 131 131 131 225 225 225 T T T 27 540-559 126 126 126 218 218 218 T T T 28 560-579 123 123 123 213 213 213 T T T 29 580-599 128 128 128 220 220 220 T T T Max gen: 132, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 131 131 131 225 225 225 F F F 3 060-079 133 133 133 228 228 228 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 130 130 130 224 224 224 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 131 131 131 225 225 225 F F F 14 280-299 131 131 131 225 225 225 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 131 131 131 225 225 225 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 133 133 133 228 228 228 F F F 19 380-399 129 129 129 222 222 222 F F F 20 400-419 130 130 130 224 224 224 F F F 21 420-439 133 133 133 228 228 228 F F F 22 440-459 131 131 131 225 225 225 F F F 23 460-479 131 131 131 225 225 225 F F F 24 480-499 133 133 133 228 228 228 F F F 25 500-519 133 133 133 228 228 228 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 133 133 133 228 228 228 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 133 133 133 228 228 228 F F F Max gen: 133, Max flush: 228 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.74014623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:00.747125675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.747179818Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.753916864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75417836Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754202487Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754211166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754222535Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Crucible 6ceb2667-ce38-4d23-ac45-90d02b745916 has session id: f7a87b05-1e5f-4398-9dec-512e1a414864","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754602947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Upstairs opts: Upstairs UUID: 6ceb2667-ce38-4d23-ac45-90d02b745916, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75465765Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Crucible stats registered with UUID: 6ceb2667-ce38-4d23-ac45-90d02b745916","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754667898Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:00.754676807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754825578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754857124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754867923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.754988998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755004586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"6ceb2667-ce38-4d23-ac45-90d02b745916 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755020304Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755028983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"0","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755038651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"1","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75504827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"2","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755058059Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755068078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755080106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"ds_connection connected from Ok(127.0.0.1:38889)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75512737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"ds_connection connected from Ok(127.0.0.1:54870)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755138179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"ds_connection connected from Ok(127.0.0.1:42431)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755148497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75591404Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"1","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.755943797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"0","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756017827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"2","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756320729Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[0]R flush_numbers[0..12]: [211, 227, 225, 228, 227, 223, 228, 224, 205, 220, 223, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756336577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[0]R generation[0..12]: [122, 132, 131, 133, 132, 130, 133, 130, 119, 128, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756347515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756358524Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[1]R flush_numbers[0..12]: [211, 227, 225, 228, 227, 223, 228, 224, 205, 220, 223, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756369093Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[1]R generation[0..12]: [122, 132, 131, 133, 132, 130, 133, 130, 119, 128, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756379601Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75639034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[2]R flush_numbers[0..12]: [211, 227, 225, 228, 227, 223, 228, 224, 205, 220, 223, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756403868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[2]R generation[0..12]: [122, 132, 131, 133, 132, 130, 133, 130, 119, 128, 130, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756414287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756425175Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756434424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756444133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756453362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756462711Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"":"downstairs","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75647228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756480948Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"0","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756490857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"1","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756500676Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"client":"2","":"downstairs","session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756510745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756519813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"6ceb2667-ce38-4d23-ac45-90d02b745916 is now active with session: f7a87b05-1e5f-4398-9dec-512e1a414864","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756529632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756538221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"session_id":"f7a87b05-1e5f-4398-9dec-512e1a414864"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.75654869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Activated sub_volume 6ceb2667-ce38-4d23-ac45-90d02b745916","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756556609Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.756564708Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.7568633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} 01/30 Write block 588 len 32768 data: 7 6 9 11 9 9 8 8 02/30 Write block 506 len 36864 data: 12 12 14 10 11 12 12 8 11 03/30 Read block 273 len 12288 04/30 Read block 196 len 32768 05/30 Read block 188 len 32768 06/30 Flush 07/30 Write block 205 len 24576 data: 16 17 19 20 20 18 08/30 Read block 360 len 28672 09/30 Read block 151 len 20480 10/30 Read block 221 len 36864 11/30 Flush 12/30 Write block 479 len 4096 data: 9 13/30 Write block 208 len 20480 data: 21 21 19 17 19 14/30 Write block 539 len 12288 data: 19 19 17 15/30 Write block 370 len 16384 data: 9 7 7 6 16/30 Write block 413 len 16384 data: 10 9 9 11 17/30 Flush 18/30 Write block 523 len 20480 data: 12 10 14 11 10 19/30 Write block 506 len 8192 data: 13 13 20/30 Write block 397 len 24576 data: 5 4 5 8 10 12 21/30 Write block 445 len 28672 data: 13 14 13 11 12 8 9 22/30 Write block 82 len 32768 data: 8 7 11 13 14 13 19 16 23/30 Write block 482 len 4096 data: 9 24/30 Write block 584 len 36864 data: 9 8 7 7 8 7 10 12 10 25/30 Write block 430 len 12288 data: 8 11 10 26/30 Read block 307 len 32768 27/30 Write block 107 len 20480 data: 12 14 14 13 12 28/30 Read block 588 len 28672 29/30 Read block 185 len 8192 30/30 Read block 121 len 36864 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 0 26 Done 5 29 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:00.764900681Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:00.765082068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1738,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 131 131 131 225 225 225 F F F 3 060-079 133 133 133 228 228 228 F F F 4 080-099 132 132 132 227 227 227 F T T <--- 5 100-119 130 130 130 223 223 223 F T T <--- 6 120-139 133 133 133 228 228 228 F F F 7 140-159 130 130 130 224 224 224 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 130 134 134 223 231 231 F F F <--- 11 220-239 133 133 133 228 228 228 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 131 131 131 225 225 225 F F F 14 280-299 131 131 131 225 225 225 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 131 131 131 225 225 225 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 133 134 134 228 231 231 F F F <--- 19 380-399 129 129 129 222 222 222 F T T <--- 20 400-419 130 134 134 224 231 231 F T T <--- 21 420-439 133 133 133 228 228 228 F T T <--- 22 440-459 131 131 131 225 225 225 F T T <--- 23 460-479 131 134 134 225 231 231 F F F <--- 24 480-499 133 133 133 228 228 228 F T T <--- 25 500-519 133 134 134 228 229 229 T T T <--- 26 520-539 133 134 134 228 231 231 F T T <--- 27 540-559 133 134 134 228 231 231 F F F <--- 28 560-579 133 133 133 228 228 228 F F F 29 580-599 133 134 134 228 229 229 T T T <--- Max gen: 134, Max flush: 231 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 131 131 131 225 225 225 F F F 3 060-079 133 133 133 228 228 228 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 130 130 130 224 224 224 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 131 131 131 225 225 225 F F F 14 280-299 131 131 131 225 225 225 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 131 131 131 225 225 225 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 134 134 134 231 231 231 F F F 19 380-399 135 135 135 232 232 232 F F F 20 400-419 135 135 135 232 232 232 F F F 21 420-439 135 135 135 232 232 232 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 134 134 134 231 231 231 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 135 135 135 232 232 232 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 F F F Max gen: 135, Max flush: 232 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.029230841Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037166434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037472286Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037498922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037507431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.03751876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Crucible 482dce02-75c5-4a7d-8a3f-2c040ebed127 has session id: c96f9890-cc24-4060-99bd-cb4b336077df","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037862446Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Upstairs opts: Upstairs UUID: 482dce02-75c5-4a7d-8a3f-2c040ebed127, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037938037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Crucible stats registered with UUID: 482dce02-75c5-4a7d-8a3f-2c040ebed127","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.037966423Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:15.037976172Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038084038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038114304Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038125253Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038290242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038314809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"482dce02-75c5-4a7d-8a3f-2c040ebed127 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038326817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038335676Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"0","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038345315Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"1","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038354724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"2","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038364153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038374041Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038408807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"ds_connection connected from Ok(127.0.0.1:56577)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038452861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"ds_connection connected from Ok(127.0.0.1:52296)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.03846416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"ds_connection connected from Ok(127.0.0.1:60101)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.038601922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039134885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"0","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039168271Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"2","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039190038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"1","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039529745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[0]R flush_numbers[0..12]: [211, 227, 225, 228, 232, 232, 228, 224, 205, 220, 231, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039551682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[0]R generation[0..12]: [122, 132, 131, 133, 135, 135, 133, 130, 119, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039562601Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039573579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[1]R flush_numbers[0..12]: [211, 227, 225, 228, 232, 232, 228, 224, 205, 220, 231, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039584308Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[1]R generation[0..12]: [122, 132, 131, 133, 135, 135, 133, 130, 119, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039594686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039605475Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[2]R flush_numbers[0..12]: [211, 227, 225, 228, 232, 232, 228, 224, 205, 220, 231, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039616114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[2]R generation[0..12]: [122, 132, 131, 133, 135, 135, 133, 130, 119, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039626622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039637451Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.03964699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039657089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039668287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039677686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"":"downstairs","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039687415Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039696154Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"0","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039705742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"1","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039715261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"client":"2","":"downstairs","session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.03972499Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039734049Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"482dce02-75c5-4a7d-8a3f-2c040ebed127 is now active with session: c96f9890-cc24-4060-99bd-cb4b336077df","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039743968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039752606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"session_id":"c96f9890-cc24-4060-99bd-cb4b336077df"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039761635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Activated sub_volume 482dce02-75c5-4a7d-8a3f-2c040ebed127","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039769434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.039777533Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.04004254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"task":"crutest"} 01/30 Flush 02/30 Write block 91 len 36864 data: 11 7 8 7 9 11 11 11 11 03/30 Read block 113 len 40960 04/30 Read block 208 len 36864 05/30 Read block 65 len 16384 06/30 Read block 444 len 24576 07/30 Flush 08/30 Write block 278 len 12288 data: 10 10 12 09/30 Write block 141 len 40960 data: 14 15 13 14 12 15 14 16 16 13 10/30 Write block 387 len 8192 data: 8 9 11/30 Read block 446 len 36864 12/30 Write block 53 len 40960 data: 12 13 12 13 15 14 13 13 11 9 13/30 Read block 124 len 40960 14/30 Write block 309 len 32768 data: 10 10 10 9 7 9 11 9 15/30 Read block 316 len 20480 16/30 Write block 370 len 12288 data: 10 8 8 17/30 Read block 341 len 40960 18/30 Write block 336 len 36864 data: 11 12 11 9 9 10 11 12 12 19/30 Write block 414 len 36864 data: 10 10 12 11 11 13 13 15 14 20/30 Read block 351 len 24576 21/30 Write block 60 len 36864 data: 14 12 10 9 8 8 8 9 7 22/30 Write block 421 len 36864 data: 16 15 15 15 16 14 13 11 12 23/30 Flush 24/30 Write block 346 len 24576 data: 17 18 16 12 10 10 25/30 Read block 540 len 40960 26/30 Flush 27/30 Write block 354 len 32768 data: 10 8 9 10 9 8 8 9 28/30 Read block 180 len 40960 29/30 Read block 361 len 16384 30/30 Read block 132 len 24576 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Done Done Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 1 23 Done 23 1 22 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1006)) Some(JobId(1025)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:15.049923896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:15.050099434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1747,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 136 131 136 235 225 235 F F F <--- 3 060-079 136 133 136 235 228 235 F F F <--- 4 080-099 136 136 136 234 234 234 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 136 130 136 235 224 235 F F F <--- 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 136 131 136 235 225 235 F T F <--- 14 280-299 136 131 136 235 225 235 F T F <--- 15 300-319 136 133 136 235 228 235 F F F <--- 16 320-339 136 131 136 235 225 235 F F F <--- 17 340-359 136 133 136 236 228 236 T F T <--- 18 360-379 136 134 136 235 231 235 T F T <--- 19 380-399 136 135 136 235 232 235 F F F <--- 20 400-419 136 135 136 235 232 235 F F F <--- 21 420-439 136 135 136 235 232 235 F F F <--- 22 440-459 135 135 135 232 232 232 F F F 23 460-479 134 134 134 231 231 231 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 135 135 135 232 232 232 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 F F F Max gen: 136, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 136 136 136 235 235 235 F F F 3 060-079 136 136 136 235 235 235 F F F 4 080-099 136 136 136 234 234 234 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 136 136 136 235 235 235 F F F 8 160-179 119 119 119 205 205 205 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 133 133 133 228 228 228 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 136 136 136 235 235 235 F F F 14 280-299 136 136 136 235 235 235 F F F 15 300-319 136 136 136 235 235 235 F F F 16 320-339 136 136 136 235 235 235 F F F 17 340-359 137 137 137 237 237 237 F F F 18 360-379 137 137 137 237 237 237 F F F 19 380-399 136 136 136 235 235 235 F F F 20 400-419 136 136 136 235 235 235 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 134 134 134 231 231 231 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 135 135 135 232 232 232 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 F F F Max gen: 137, Max flush: 237 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.300772307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.308836254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309138566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309163443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309174481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.3091861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Crucible 825d34ff-c05e-44c7-bcec-c134794bdea9 has session id: 2c93fc98-25f1-4fc1-8758-f361314447fd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309569231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Upstairs opts: Upstairs UUID: 825d34ff-c05e-44c7-bcec-c134794bdea9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309598938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Crucible stats registered with UUID: 825d34ff-c05e-44c7-bcec-c134794bdea9","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309617245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:29.309626674Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309761287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309778055Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309788094Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309950283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309967811Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"825d34ff-c05e-44c7-bcec-c134794bdea9 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309980679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309989378Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"0","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.309998977Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"1","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310010885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"2","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310020404Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310030173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310040022Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:42241)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310141589Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33232)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310169085Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:56327)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310216449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310798195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"0","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310817473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"2","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.310857958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"1","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311144772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[0]R flush_numbers[0..12]: [211, 227, 235, 235, 234, 232, 228, 235, 205, 220, 231, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311162569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[0]R generation[0..12]: [122, 132, 136, 136, 136, 135, 133, 136, 119, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311173288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311184356Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[1]R flush_numbers[0..12]: [211, 227, 235, 235, 234, 232, 228, 235, 205, 220, 231, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311197305Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[1]R generation[0..12]: [122, 132, 136, 136, 136, 135, 133, 136, 119, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311208103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311218892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[2]R flush_numbers[0..12]: [211, 227, 235, 235, 234, 232, 228, 235, 205, 220, 231, 228]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311229531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[2]R generation[0..12]: [122, 132, 136, 136, 136, 135, 133, 136, 119, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311239949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311250658Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311260037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311269726Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311279124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311288483Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"":"downstairs","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311298102Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311306881Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"0","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.31131653Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"1","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311326338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"client":"2","":"downstairs","session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311336147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311346566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"825d34ff-c05e-44c7-bcec-c134794bdea9 is now active with session: 2c93fc98-25f1-4fc1-8758-f361314447fd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311356365Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311364974Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"session_id":"2c93fc98-25f1-4fc1-8758-f361314447fd"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311374092Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Activated sub_volume 825d34ff-c05e-44c7-bcec-c134794bdea9","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311382051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.31139012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.311713369Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"task":"crutest"} 01/30 Write block 368 len 24576 data: 8 9 11 9 9 7 02/30 Flush 03/30 Flush 04/30 Write block 347 len 4096 data: 19 05/30 Read block 360 len 40960 06/30 Write block 169 len 28672 data: 14 15 15 16 15 14 15 07/30 Flush 08/30 Write block 387 len 8192 data: 9 10 09/30 Read block 172 len 16384 10/30 Write block 214 len 36864 data: 16 13 12 11 13 14 16 16 12 11/30 Read block 134 len 12288 12/30 Write block 96 len 16384 data: 12 12 12 12 13/30 Write block 468 len 28672 data: 10 7 6 6 7 6 7 14/30 Write block 58 len 32768 data: 15 14 15 13 11 10 9 9 15/30 Write block 43 len 36864 data: 8 11 9 12 12 12 16 14 15 16/30 Write block 256 len 4096 data: 10 17/30 Write block 311 len 40960 data: 11 10 8 10 12 10 8 8 7 10 18/30 Read block 42 len 8192 19/30 Read block 305 len 28672 20/30 Flush 21/30 Write block 476 len 36864 data: 7 8 7 10 10 9 10 10 11 22/30 Read block 85 len 4096 23/30 Write block 381 len 32768 data: 8 8 8 9 8 7 10 11 24/30 Read block 61 len 8192 25/30 Write block 369 len 24576 data: 10 12 10 10 8 8 26/30 Write block 253 len 40960 data: 10 9 9 11 8 10 10 9 8 9 27/30 Read block 149 len 8192 28/30 Read block 128 len 32768 29/30 Write block 12 len 28672 data: 14 10 11 12 10 11 9 30/30 Write block 589 len 36864 data: 8 11 13 11 10 9 9 7 5 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 21 2 25 Done 21 2 21 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1006)) Some(JobId(1019)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:29.320364112Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:29.320543909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1756,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 T F T <--- 1 020-039 132 132 132 227 227 227 F F F 2 040-059 138 136 138 241 235 241 F F F <--- 3 060-079 138 136 138 241 235 241 F F F <--- 4 080-099 138 136 138 241 234 241 F F F <--- 5 100-119 135 135 135 232 232 232 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 136 136 136 235 235 235 F F F 8 160-179 138 138 138 240 240 240 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 138 134 138 241 231 241 F F F <--- 11 220-239 138 133 138 241 228 241 F F F <--- 12 240-259 138 133 138 241 228 241 T F T <--- 13 260-279 136 136 136 235 235 235 T F T <--- 14 280-299 136 136 136 235 235 235 F F F 15 300-319 138 136 138 241 235 241 F F F <--- 16 320-339 138 136 138 241 235 241 F F F <--- 17 340-359 138 138 138 240 240 240 F F F 18 360-379 138 138 138 238 238 238 T F T <--- 19 380-399 138 136 138 241 235 241 T T T <--- 20 400-419 136 136 136 235 235 235 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 138 134 138 241 231 241 T F T <--- 24 480-499 135 135 135 232 232 232 T F T <--- 25 500-519 135 135 135 232 232 232 F F F 26 520-539 135 135 135 232 232 232 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 135 135 135 232 232 232 T F T <--- Max gen: 138, Max flush: 241 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 242 242 242 F F F 1 020-039 132 132 132 227 227 227 F F F 2 040-059 138 138 138 241 241 241 F F F 3 060-079 138 138 138 241 241 241 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 133 133 133 228 228 228 F F F 7 140-159 136 136 136 235 235 235 F F F 8 160-179 138 138 138 240 240 240 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 138 138 138 241 241 241 F F F 11 220-239 138 138 138 241 241 241 F F F 12 240-259 139 139 139 242 242 242 F F F 13 260-279 139 139 139 242 242 242 F F F 14 280-299 136 136 136 235 235 235 F F F 15 300-319 138 138 138 241 241 241 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 138 138 138 240 240 240 F F F 18 360-379 139 139 139 242 242 242 F F F 19 380-399 139 139 139 242 242 242 F F F 20 400-419 136 136 136 235 235 235 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 139 139 139 242 242 242 F F F 24 480-499 139 139 139 242 242 242 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 135 135 135 232 232 232 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 139, Max flush: 242 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.578693854Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.586780488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587111016Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587144192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587153251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.58716487Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Crucible 9a9baf92-0a70-45e8-8e81-f88097760fc2 has session id: 74aecf40-a4fa-4021-84c3-95645a548080","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587513305Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Upstairs opts: Upstairs UUID: 9a9baf92-0a70-45e8-8e81-f88097760fc2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587530233Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Crucible stats registered with UUID: 9a9baf92-0a70-45e8-8e81-f88097760fc2","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587544621Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:43.5875536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587703191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587719529Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587729448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587858912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587875969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"9a9baf92-0a70-45e8-8e81-f88097760fc2 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587892817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587901416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"0","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587911035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"1","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587920434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"2","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587929943Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.587939761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.58794964Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"ds_connection connected from Ok(127.0.0.1:45405)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.588019951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"ds_connection connected from Ok(127.0.0.1:43911)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.588061516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"ds_connection connected from Ok(127.0.0.1:50754)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.588103681Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.588833198Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"2","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.588859295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"1","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.588870633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"0","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589196562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[0]R flush_numbers[0..12]: [242, 227, 241, 241, 241, 232, 228, 235, 240, 220, 241, 241]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589231558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[0]R generation[0..12]: [139, 132, 138, 138, 138, 135, 133, 136, 138, 128, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589242586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589253605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[1]R flush_numbers[0..12]: [242, 227, 241, 241, 241, 232, 228, 235, 240, 220, 241, 241]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589264073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[1]R generation[0..12]: [139, 132, 138, 138, 138, 135, 133, 136, 138, 128, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589274342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589285151Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[2]R flush_numbers[0..12]: [242, 227, 241, 241, 241, 232, 228, 235, 240, 220, 241, 241]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589295679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[2]R generation[0..12]: [139, 132, 138, 138, 138, 135, 133, 136, 138, 128, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589306088Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589316817Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589328165Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589337884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589347133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589356442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"":"downstairs","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589365981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589374569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"0","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589384238Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"1","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589393787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"client":"2","":"downstairs","session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589403556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589412785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"9a9baf92-0a70-45e8-8e81-f88097760fc2 is now active with session: 74aecf40-a4fa-4021-84c3-95645a548080","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589422773Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589431562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"session_id":"74aecf40-a4fa-4021-84c3-95645a548080"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.58944498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Activated sub_volume 9a9baf92-0a70-45e8-8e81-f88097760fc2","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589453439Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589461518Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.589707567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"task":"crutest"} 01/30 Write block 414 len 4096 data: 11 02/30 Read block 364 len 24576 03/30 Write block 169 len 28672 data: 15 16 16 17 16 15 16 04/30 Write block 121 len 8192 data: 9 14 05/30 Write block 55 len 4096 data: 13 06/30 Write block 271 len 36864 data: 7 7 8 7 10 11 12 11 11 07/30 Read block 69 len 24576 08/30 Read block 416 len 28672 09/30 Write block 24 len 32768 data: 4 5 5 9 9 9 10 10 10/30 Read block 155 len 24576 11/30 Read block 278 len 24576 12/30 Read block 473 len 12288 13/30 Read block 377 len 40960 14/30 Read block 444 len 36864 15/30 Write block 404 len 36864 data: 12 10 9 10 10 9 7 8 10 16/30 Write block 130 len 32768 data: 15 19 21 16 17 14 12 10 17/30 Write block 245 len 24576 data: 10 9 11 15 14 8 18/30 Read block 369 len 4096 19/30 Write block 221 len 8192 data: 17 13 20/30 Write block 161 len 24576 data: 10 9 12 13 9 6 21/30 Write block 117 len 40960 data: 6 9 8 9 10 15 16 17 17 18 22/30 Read block 375 len 8192 23/30 Write block 231 len 20480 data: 9 7 7 7 9 24/30 Read block 101 len 12288 25/30 Write block 359 len 36864 data: 9 9 10 10 11 11 12 12 11 26/30 Read block 242 len 8192 27/30 Read block 470 len 40960 28/30 Read block 563 len 28672 29/30 Write block 522 len 28672 data: 12 13 11 15 12 11 9 30/30 Write block 256 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 20 2 24 Done 28 10 28 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:43.597746608Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:43.597972139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1765,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 242 242 242 F F F 1 020-039 132 132 132 227 227 227 T T T 2 040-059 138 138 138 241 241 241 T T T 3 060-079 138 138 138 241 241 241 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 135 135 135 232 232 232 T F T <--- 6 120-139 133 133 133 228 228 228 T T T 7 140-159 136 136 136 235 235 235 F F F 8 160-179 138 138 138 240 240 240 T T T 9 180-199 128 128 128 220 220 220 F F F 10 200-219 138 138 138 241 241 241 F F F 11 220-239 138 138 138 241 241 241 T F T <--- 12 240-259 139 139 139 242 242 242 T F T <--- 13 260-279 139 139 139 242 242 242 T T T 14 280-299 136 136 136 235 235 235 F F F 15 300-319 138 138 138 241 241 241 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 138 138 138 240 240 240 T F T <--- 18 360-379 139 139 139 242 242 242 T F T <--- 19 380-399 139 139 139 242 242 242 F F F 20 400-419 136 136 136 235 235 235 T T T 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 139 139 139 242 242 242 F F F 24 480-499 139 139 139 242 242 242 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 135 135 135 232 232 232 T F T <--- 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 139, Max flush: 242 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 242 242 242 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 138 138 138 241 241 241 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 140 140 140 243 243 243 F F F 6 120-139 140 140 140 243 243 243 F F F 7 140-159 136 136 136 235 235 235 F F F 8 160-179 140 140 140 243 243 243 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 138 138 138 241 241 241 F F F 11 220-239 140 140 140 243 243 243 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 136 136 136 235 235 235 F F F 15 300-319 138 138 138 241 241 241 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 140 140 140 243 243 243 F F F 19 380-399 139 139 139 242 242 242 F F F 20 400-419 140 140 140 243 243 243 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 139 139 139 242 242 242 F F F 24 480-499 139 139 139 242 242 242 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 140 140 140 243 243 243 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 140, Max flush: 243 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.859292901Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:57.866296263Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.866342687Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873269668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873528716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873552323Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873561011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.87357243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Crucible 2bc62591-e6c8-4fc9-b7e3-3a0e525e70fc has session id: 98972b1e-d4d2-4456-aca0-83af9aa59ce5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873876311Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Upstairs opts: Upstairs UUID: 2bc62591-e6c8-4fc9-b7e3-3a0e525e70fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873935864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Crucible stats registered with UUID: 2bc62591-e6c8-4fc9-b7e3-3a0e525e70fc","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.873947682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:57.873956951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874106012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.87412246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874134949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874270401Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874286839Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"2bc62591-e6c8-4fc9-b7e3-3a0e525e70fc active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874320655Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874329834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"0","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874339423Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"1","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874349071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"2","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.87435864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874368639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874378498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37710)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874578632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46219)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874628606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39274)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.874650623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875297331Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"1","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875322998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"0","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875334736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"2","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875687152Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[0]R flush_numbers[0..12]: [242, 243, 243, 241, 241, 243, 243, 235, 243, 220, 241, 243]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875707779Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[0]R generation[0..12]: [139, 140, 140, 138, 138, 140, 140, 136, 140, 128, 138, 140]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875718568Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875729376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[1]R flush_numbers[0..12]: [242, 243, 243, 241, 241, 243, 243, 235, 243, 220, 241, 243]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875739875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[1]R generation[0..12]: [139, 140, 140, 138, 138, 140, 140, 136, 140, 128, 138, 140]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875750464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875761292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[2]R flush_numbers[0..12]: [242, 243, 243, 241, 241, 243, 243, 235, 243, 220, 241, 243]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875771811Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[2]R generation[0..12]: [139, 140, 140, 138, 138, 140, 140, 136, 140, 128, 138, 140]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.87578216Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875793228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875802757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875812596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875821895Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875831244Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"":"downstairs","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875842922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875851571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"0","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.87586103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"1","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875870569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"client":"2","":"downstairs","session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875880287Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875889276Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"2bc62591-e6c8-4fc9-b7e3-3a0e525e70fc is now active with session: 98972b1e-d4d2-4456-aca0-83af9aa59ce5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875899105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875907764Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"session_id":"98972b1e-d4d2-4456-aca0-83af9aa59ce5"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875916823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Activated sub_volume 2bc62591-e6c8-4fc9-b7e3-3a0e525e70fc","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875924762Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.875932801Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.87625503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} 01/30 Flush 02/30 Write block 280 len 32768 data: 13 14 13 15 11 12 14 12 03/30 Read block 403 len 20480 04/30 Write block 502 len 28672 data: 9 11 12 15 14 14 15 05/30 Write block 170 len 36864 data: 17 17 18 17 16 17 14 13 8 06/30 Read block 444 len 16384 07/30 Read block 437 len 24576 08/30 Read block 37 len 36864 09/30 Read block 458 len 40960 10/30 Read block 371 len 8192 11/30 Write block 127 len 8192 data: 17 14 12/30 Read block 551 len 32768 13/30 Read block 455 len 28672 14/30 Read block 567 len 12288 15/30 Write block 68 len 4096 data: 8 16/30 Read block 149 len 36864 17/30 Flush 18/30 Flush 19/30 Write block 520 len 40960 data: 15 13 13 14 12 16 13 12 10 9 20/30 Write block 217 len 36864 data: 12 14 15 17 18 14 10 10 9 21/30 Write block 412 len 32768 data: 11 11 12 11 13 12 12 14 22/30 Read block 186 len 16384 23/30 Write block 382 len 24576 data: 9 9 10 9 8 11 24/30 Write block 461 len 32768 data: 11 13 17 13 14 12 10 11 25/30 Read block 88 len 40960 26/30 Read block 434 len 28672 27/30 Read block 55 len 8192 28/30 Write block 400 len 16384 data: 9 11 13 12 29/30 Read block 74 len 8192 30/30 Read block 548 len 28672 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:01:57.885377443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:01:57.88555655Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1774,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 242 242 242 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 138 142 142 241 245 245 F F F <--- 4 080-099 138 138 138 241 241 241 F F F 5 100-119 140 140 140 243 243 243 F F F 6 120-139 140 142 142 243 245 245 F F F <--- 7 140-159 136 136 136 235 235 235 F F F 8 160-179 140 142 142 243 245 245 F F F <--- 9 180-199 128 128 128 220 220 220 F F F 10 200-219 138 138 138 241 241 241 F T T <--- 11 220-239 140 140 140 243 243 243 F T T <--- 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 136 142 142 235 245 245 F F F <--- 15 300-319 138 138 138 241 241 241 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 140 140 140 243 243 243 F F F 19 380-399 139 139 139 242 242 242 F T T <--- 20 400-419 140 140 140 243 243 243 F T T <--- 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 139 139 139 242 242 242 F T T <--- 24 480-499 139 139 139 242 242 242 F F F 25 500-519 135 142 142 232 245 245 F F F <--- 26 520-539 140 140 140 243 243 243 F T T <--- 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 142, Max flush: 245 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 242 242 242 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 140 140 140 243 243 243 F F F 6 120-139 142 142 142 245 245 245 F F F 7 140-159 136 136 136 235 235 235 F F F 8 160-179 142 142 142 245 245 245 F F F 9 180-199 128 128 128 220 220 220 F F F 10 200-219 143 143 143 246 246 246 F F F 11 220-239 143 143 143 246 246 246 F F F 12 240-259 140 140 140 243 243 243 F F F 13 260-279 140 140 140 243 243 243 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 138 138 138 241 241 241 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 140 140 140 243 243 243 F F F 19 380-399 143 143 143 246 246 246 F F F 20 400-419 143 143 143 246 246 246 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 143 143 143 246 246 246 F F F 24 480-499 139 139 139 242 242 242 F F F 25 500-519 142 142 142 245 245 245 F F F 26 520-539 143 143 143 246 246 246 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 143, Max flush: 246 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.137616037Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:12.144592862Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.144645496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.151335247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.151587155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.151612622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.151621091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.151632619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Crucible 48d1a666-48c5-4c57-8b79-d18841c342bd has session id: 8f25d9c8-538f-4130-8cef-8ebf4c73b5d3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.151974686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Upstairs opts: Upstairs UUID: 48d1a666-48c5-4c57-8b79-d18841c342bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152005122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Crucible stats registered with UUID: 48d1a666-48c5-4c57-8b79-d18841c342bd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152016271Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:12.15202543Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152160492Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.15217651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152186619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152328641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152345089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"48d1a666-48c5-4c57-8b79-d18841c342bd active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152359307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152368046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"0","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152379885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"1","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152389493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"2","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152399012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152408861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152451266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45275)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152472403Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64395)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.152483471Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45137)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.15249534Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153174034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"1","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153197201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"2","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.15320855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"0","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153476386Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[0]R flush_numbers[0..12]: [242, 243, 243, 245, 241, 243, 245, 235, 245, 220, 246, 246]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153494843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[0]R generation[0..12]: [139, 140, 140, 142, 138, 140, 142, 136, 142, 128, 143, 143]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153505602Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.15351658Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[1]R flush_numbers[0..12]: [242, 243, 243, 245, 241, 243, 245, 235, 245, 220, 246, 246]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153529579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[1]R generation[0..12]: [139, 140, 140, 142, 138, 140, 142, 136, 142, 128, 143, 143]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153539987Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153550906Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[2]R flush_numbers[0..12]: [242, 243, 243, 245, 241, 243, 245, 235, 245, 220, 246, 246]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153561405Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[2]R generation[0..12]: [139, 140, 140, 142, 138, 140, 142, 136, 142, 128, 143, 143]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153571763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153582682Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153591951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.15360183Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153611058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153620367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"":"downstairs","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153630006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153638775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"0","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153648564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"1","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153658142Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"client":"2","":"downstairs","session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153669521Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.15367866Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"48d1a666-48c5-4c57-8b79-d18841c342bd is now active with session: 8f25d9c8-538f-4130-8cef-8ebf4c73b5d3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153688589Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153697248Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"session_id":"8f25d9c8-538f-4130-8cef-8ebf4c73b5d3"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153706286Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Activated sub_volume 48d1a666-48c5-4c57-8b79-d18841c342bd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153714245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.153722224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.154050853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} 01/30 Write block 11 len 4096 data: 12 02/30 Read block 181 len 8192 03/30 Read block 66 len 16384 04/30 Read block 525 len 8192 05/30 Write block 370 len 32768 data: 13 11 11 9 9 11 12 12 06/30 Read block 38 len 28672 07/30 Flush 08/30 Flush 09/30 Write block 105 len 12288 data: 10 9 13 10/30 Read block 382 len 20480 11/30 Read block 110 len 40960 12/30 Read block 229 len 28672 13/30 Read block 531 len 32768 14/30 Read block 144 len 40960 15/30 Write block 153 len 12288 data: 14 14 13 16/30 Read block 334 len 16384 17/30 Write block 304 len 20480 data: 12 11 12 13 14 18/30 Read block 380 len 12288 19/30 Flush 20/30 Flush 21/30 Write block 504 len 16384 data: 13 16 15 15 22/30 Write block 240 len 32768 data: 9 6 8 6 7 11 10 12 23/30 Read block 592 len 24576 24/30 Read block 325 len 12288 25/30 Read block 479 len 36864 26/30 Read block 198 len 4096 27/30 Write block 218 len 36864 data: 15 16 18 19 15 11 11 10 8 28/30 Write block 189 len 28672 data: 14 11 10 13 15 12 11 29/30 Write block 270 len 32768 data: 8 8 8 9 8 11 12 13 30/30 Write block 533 len 16384 data: 14 14 14 15 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 3 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 13 3 3 19 Done 9 19 19 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:12.162647392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:12.162816511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1783,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 140 144 144 243 249 249 T F F <--- 6 120-139 142 142 142 245 245 245 F F F 7 140-159 136 144 144 235 249 249 T F F <--- 8 160-179 142 142 142 245 245 245 F F F 9 180-199 128 128 128 220 220 220 F T T <--- 10 200-219 143 143 143 246 246 246 F T T <--- 11 220-239 143 143 143 246 246 246 F T T <--- 12 240-259 140 140 140 243 243 243 F T T <--- 13 260-279 140 140 140 243 243 243 F T T <--- 14 280-299 142 142 142 245 245 245 F F F 15 300-319 138 144 144 241 249 249 T F F <--- 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 144 144 144 247 247 247 F F F 19 380-399 143 143 143 246 246 246 F F F 20 400-419 143 143 143 246 246 246 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 143 143 143 246 246 246 F F F 24 480-499 139 139 139 242 242 242 F F F 25 500-519 142 142 142 245 245 245 F T T <--- 26 520-539 143 143 143 246 246 246 F T T <--- 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 144, Max flush: 249 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 144 144 144 249 249 249 F F F 6 120-139 142 142 142 245 245 245 F F F 7 140-159 144 144 144 249 249 249 F F F 8 160-179 142 142 142 245 245 245 F F F 9 180-199 145 145 145 250 250 250 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 145 145 145 250 250 250 F F F 12 240-259 145 145 145 250 250 250 F F F 13 260-279 145 145 145 250 250 250 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 144 144 144 247 247 247 F F F 19 380-399 143 143 143 246 246 246 F F F 20 400-419 143 143 143 246 246 246 F F F 21 420-439 136 136 136 235 235 235 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 143 143 143 246 246 246 F F F 24 480-499 139 139 139 242 242 242 F F F 25 500-519 145 145 145 250 250 250 F F F 26 520-539 145 145 145 250 250 250 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 145, Max flush: 250 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.411299931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.419297347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.41958825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.419613597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.419622196Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.419633775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Crucible 1050402e-d480-45a7-943c-c27c3b5117cc has session id: 2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.41998195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Upstairs opts: Upstairs UUID: 1050402e-d480-45a7-943c-c27c3b5117cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420012227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Crucible stats registered with UUID: 1050402e-d480-45a7-943c-c27c3b5117cc","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420023815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:26.420039823Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420201703Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420233259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420247687Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.42038268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420400927Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"1050402e-d480-45a7-943c-c27c3b5117cc active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420413566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420422375Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"0","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420431933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"1","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420441522Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"2","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420450961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.42046082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420470828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47621)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.42053564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33444)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420551918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49250)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.420562227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421197266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"0","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.42124532Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"1","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421295364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"2","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421546982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[0]R flush_numbers[0..12]: [247, 243, 243, 245, 241, 249, 245, 249, 245, 250, 250, 250]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421573918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[0]R generation[0..12]: [144, 140, 140, 142, 138, 144, 142, 144, 142, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421585137Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421596186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[1]R flush_numbers[0..12]: [247, 243, 243, 245, 241, 249, 245, 249, 245, 250, 250, 250]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421606504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[1]R generation[0..12]: [144, 140, 140, 142, 138, 144, 142, 144, 142, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421616903Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421627632Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[2]R flush_numbers[0..12]: [247, 243, 243, 245, 241, 249, 245, 249, 245, 250, 250, 250]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[2]R generation[0..12]: [144, 140, 140, 142, 138, 144, 142, 144, 142, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421648729Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421659428Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421670926Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421680675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421689874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421699253Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"":"downstairs","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421708801Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.42171745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"0","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421727049Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"1","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421736678Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"client":"2","":"downstairs","session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421746727Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421755815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"1050402e-d480-45a7-943c-c27c3b5117cc is now active with session: 2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421765634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421774283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"session_id":"2d9a73b2-e4f4-4a81-bf77-046c6b08b6d4"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421783192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Activated sub_volume 1050402e-d480-45a7-943c-c27c3b5117cc","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.421791181Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.42179912Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.422126188Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"task":"crutest"} 01/30 Flush 02/30 Write block 492 len 4096 data: 9 03/30 Write block 503 len 12288 data: 12 14 17 04/30 Read block 106 len 40960 05/30 Read block 108 len 8192 06/30 Read block 57 len 16384 07/30 Write block 158 len 4096 data: 11 08/30 Flush 09/30 Write block 546 len 20480 data: 9 9 8 9 10 10/30 Flush 11/30 Write block 391 len 40960 data: 12 13 11 10 7 5 6 5 6 10 12/30 Read block 126 len 4096 13/30 Read block 307 len 8192 14/30 Read block 599 len 4096 15/30 Write block 37 len 12288 data: 7 6 6 16/30 Write block 106 len 12288 data: 10 14 15 17/30 Write block 258 len 20480 data: 11 11 10 9 10 18/30 Write block 183 len 12288 data: 16 14 13 19/30 Flush 20/30 Write block 432 len 28672 data: 11 10 10 9 9 6 7 21/30 Read block 323 len 28672 22/30 Write block 373 len 20480 data: 10 10 12 13 13 23/30 Read block 286 len 8192 24/30 Read block 244 len 8192 25/30 Read block 441 len 8192 26/30 Read block 596 len 4096 27/30 Read block 51 len 8192 28/30 Read block 270 len 28672 29/30 Read block 463 len 36864 30/30 Read block 273 len 16384 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:26.42960179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:26.429781168Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1792,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 140 146 254 243 254 F F F <--- 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 146 144 146 254 249 254 F F F <--- 6 120-139 142 142 142 245 245 245 F F F 7 140-159 146 144 146 252 249 252 F F F <--- 8 160-179 142 142 142 245 245 245 F F F 9 180-199 146 145 146 254 250 254 F F F <--- 10 200-219 145 145 145 250 250 250 F F F 11 220-239 145 145 145 250 250 250 F F F 12 240-259 146 145 146 254 250 254 F F F <--- 13 260-279 146 145 146 254 250 254 F F F <--- 14 280-299 142 142 142 245 245 245 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 144 144 144 247 247 247 T F T <--- 19 380-399 146 143 146 254 246 254 F F F <--- 20 400-419 146 143 146 254 246 254 F F F <--- 21 420-439 136 136 136 235 235 235 T F T <--- 22 440-459 135 135 135 232 232 232 F F F 23 460-479 143 143 143 246 246 246 F F F 24 480-499 146 139 146 252 242 252 F F F <--- 25 500-519 146 145 146 252 250 252 F F F <--- 26 520-539 145 145 145 250 250 250 F F F 27 540-559 146 134 146 253 231 253 F F F <--- 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 146, Max flush: 254 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 146 146 254 254 254 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 146 146 146 254 254 254 F F F 6 120-139 142 142 142 245 245 245 F F F 7 140-159 146 146 146 252 252 252 F F F 8 160-179 142 142 142 245 245 245 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 145 145 145 250 250 250 F F F 12 240-259 146 146 146 254 254 254 F F F 13 260-279 146 146 146 254 254 254 F F F 14 280-299 142 142 142 245 245 245 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 147 147 147 255 255 255 F F F 19 380-399 146 146 146 254 254 254 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 147 147 147 255 255 255 F F F 22 440-459 135 135 135 232 232 232 F F F 23 460-479 143 143 143 246 246 246 F F F 24 480-499 146 146 146 252 252 252 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 145 145 145 250 250 250 F F F 27 540-559 146 146 146 253 253 253 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 147, Max flush: 255 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.681526974Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.689469917Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.689771889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.689807314Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.689816593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.689828172Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Crucible 7a6c56ad-a2d3-479e-8166-a0305c4365f1 has session id: 6365a421-98a3-4459-81fd-7b412a10448b","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690225481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Upstairs opts: Upstairs UUID: 7a6c56ad-a2d3-479e-8166-a0305c4365f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690253088Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Crucible stats registered with UUID: 7a6c56ad-a2d3-479e-8166-a0305c4365f1","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690266636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:40.690275905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690400989Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690422376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690432505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690581896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690605533Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"7a6c56ad-a2d3-479e-8166-a0305c4365f1 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690618221Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.69062933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"0","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690639229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"1","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690649007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"2","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690658406Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690668175Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690678034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64727)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690733227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33762)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690758704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49290)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.690769712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691424999Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"2","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691464064Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"1","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691528546Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"0","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691795432Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[0]R flush_numbers[0..12]: [247, 254, 243, 245, 241, 254, 245, 252, 245, 254, 250, 250]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.69181445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[0]R generation[0..12]: [144, 146, 140, 142, 138, 146, 142, 146, 142, 146, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691825438Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691839376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[1]R flush_numbers[0..12]: [247, 254, 243, 245, 241, 254, 245, 252, 245, 254, 250, 250]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691850145Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[1]R generation[0..12]: [144, 146, 140, 142, 138, 146, 142, 146, 142, 146, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691860564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691871292Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[2]R flush_numbers[0..12]: [247, 254, 243, 245, 241, 254, 245, 252, 245, 254, 250, 250]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691881621Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[2]R generation[0..12]: [144, 146, 140, 142, 138, 146, 142, 146, 142, 146, 145, 145]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.69189193Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691902728Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691912047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691921706Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691930905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691940224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"":"downstairs","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691949932Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691958571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"0","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.69196793Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"1","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691978869Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"client":"2","":"downstairs","session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691988698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.691997586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"7a6c56ad-a2d3-479e-8166-a0305c4365f1 is now active with session: 6365a421-98a3-4459-81fd-7b412a10448b","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.692007715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.692016394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"session_id":"6365a421-98a3-4459-81fd-7b412a10448b"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.692025263Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Activated sub_volume 7a6c56ad-a2d3-479e-8166-a0305c4365f1","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.692033182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.692041121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.692353271Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"task":"crutest"} 01/30 Write block 456 len 32768 data: 12 12 11 12 12 12 14 18 02/30 Write block 15 len 8192 data: 13 11 03/30 Write block 238 len 40960 data: 11 8 10 7 9 7 8 12 11 13 04/30 Read block 559 len 4096 05/30 Read block 423 len 8192 06/30 Read block 443 len 28672 07/30 Read block 430 len 28672 08/30 Write block 115 len 8192 data: 9 8 09/30 Write block 504 len 28672 data: 15 18 16 16 16 11 12 10/30 Read block 357 len 8192 11/30 Write block 279 len 32768 data: 12 14 15 14 16 12 13 15 12/30 Read block 239 len 8192 13/30 Write block 517 len 28672 data: 14 11 13 16 14 14 15 14/30 Write block 390 len 40960 data: 14 13 14 12 11 8 6 7 6 7 15/30 Write block 544 len 32768 data: 12 11 10 10 9 10 11 9 16/30 Flush 17/30 Write block 408 len 12288 data: 11 10 8 18/30 Write block 149 len 4096 data: 17 19/30 Read block 238 len 28672 20/30 Read block 122 len 40960 21/30 Read block 454 len 32768 22/30 Read block 356 len 20480 23/30 Read block 232 len 40960 24/30 Read block 181 len 36864 25/30 Read block 248 len 36864 26/30 Write block 272 len 36864 data: 9 10 9 12 13 14 12 13 15 27/30 Write block 122 len 36864 data: 16 17 18 18 19 18 15 17 16 28/30 Read block 384 len 8192 29/30 Write block 220 len 32768 data: 19 20 16 12 12 11 9 8 30/30 Write block 8 len 36864 data: 5 4 8 13 15 11 12 14 12 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:40.700974668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:40.701156855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1801,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 144 148 256 247 256 T F T <--- 1 020-039 146 146 146 254 254 254 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 148 146 148 256 254 256 F F F <--- 6 120-139 142 142 142 245 245 245 T F T <--- 7 140-159 146 146 146 252 252 252 T F T <--- 8 160-179 142 142 142 245 245 245 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 148 145 148 256 250 256 T F T <--- 12 240-259 148 146 148 256 254 256 F F F <--- 13 260-279 148 146 148 256 254 256 T F T <--- 14 280-299 148 142 148 256 245 256 T F T <--- 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 147 147 147 255 255 255 F F F 19 380-399 148 146 148 256 254 256 F F F <--- 20 400-419 146 146 146 254 254 254 T F T <--- 21 420-439 147 147 147 255 255 255 F F F 22 440-459 148 135 148 256 232 256 F F F <--- 23 460-479 148 143 148 256 246 256 F F F <--- 24 480-499 146 146 146 252 252 252 F F F 25 500-519 148 146 148 256 252 256 F F F <--- 26 520-539 148 145 148 256 250 256 F F F <--- 27 540-559 148 146 148 256 253 256 F F F <--- 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 148, Max flush: 256 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 146 146 146 254 254 254 F F F 2 040-059 140 140 140 243 243 243 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 149 149 149 257 257 257 F F F 7 140-159 149 149 149 257 257 257 F F F 8 160-179 142 142 142 245 245 245 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 149 149 149 257 257 257 F F F 14 280-299 149 149 149 257 257 257 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F F F 18 360-379 147 147 147 255 255 255 F F F 19 380-399 148 148 148 256 256 256 F F F 20 400-419 149 149 149 257 257 257 F F F 21 420-439 147 147 147 255 255 255 F F F 22 440-459 148 148 148 256 256 256 F F F 23 460-479 148 148 148 256 256 256 F F F 24 480-499 146 146 146 252 252 252 F F F 25 500-519 148 148 148 256 256 256 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 149, Max flush: 257 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.97198859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:54.978971085Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.979024968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.985811797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986087082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986111369Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986122488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986133966Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Crucible 170ce37a-307b-43c4-b420-f4aef983252f has session id: 26ed5c11-f140-419d-8cce-ad74e466250b","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986487592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Upstairs opts: Upstairs UUID: 170ce37a-307b-43c4-b420-f4aef983252f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986526277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Crucible stats registered with UUID: 170ce37a-307b-43c4-b420-f4aef983252f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986540555Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:54.986549704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986698485Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986715203Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986725212Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986874763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.9868913Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"170ce37a-307b-43c4-b420-f4aef983252f active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986914827Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986924126Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"0","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986933815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"1","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986943354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"2","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986952813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.986964761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.98697464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46237)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.987008696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50482)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.987030503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64964)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.987040901Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.987617128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"2","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.987647485Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"1","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.987757781Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"0","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988007389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[0]R flush_numbers[0..12]: [257, 254, 243, 245, 241, 256, 257, 257, 245, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988026426Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[0]R generation[0..12]: [149, 146, 140, 142, 138, 148, 149, 149, 142, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988037245Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988048124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[1]R flush_numbers[0..12]: [257, 254, 243, 245, 241, 256, 257, 257, 245, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988059062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[1]R generation[0..12]: [149, 146, 140, 142, 138, 148, 149, 149, 142, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988069661Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988082749Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[2]R flush_numbers[0..12]: [257, 254, 243, 245, 241, 256, 257, 257, 245, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988093408Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[2]R generation[0..12]: [149, 146, 140, 142, 138, 148, 149, 149, 142, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988103787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988114575Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988123874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988133583Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988142902Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988152231Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"":"downstairs","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988161939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988170718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"0","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988180297Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"1","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988189936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"client":"2","":"downstairs","session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988199725Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988208953Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"170ce37a-307b-43c4-b420-f4aef983252f is now active with session: 26ed5c11-f140-419d-8cce-ad74e466250b","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988218892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988228871Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"session_id":"26ed5c11-f140-419d-8cce-ad74e466250b"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-10-01T01:02:54.98823796Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Activated sub_volume 170ce37a-307b-43c4-b420-f4aef983252f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988250428Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988258567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.988500306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} 01/30 Write block 507 len 36864 data: 17 17 12 13 13 13 9 12 13 02/30 Read block 395 len 32768 03/30 Read block 218 len 40960 04/30 Read block 226 len 12288 05/30 Write block 144 len 20480 data: 15 13 16 15 17 06/30 Write block 486 len 8192 data: 9 10 07/30 Flush 08/30 Flush 09/30 Write block 459 len 20480 data: 13 13 13 15 19 10/30 Write block 408 len 40960 data: 12 11 9 9 12 12 13 12 14 13 11/30 Write block 136 len 8192 data: 13 11 12/30 Write block 112 len 4096 data: 9 13/30 Read block 534 len 36864 14/30 Read block 324 len 20480 15/30 Read block 279 len 8192 16/30 Read block 502 len 20480 17/30 Read block 36 len 4096 18/30 Read block 344 len 40960 19/30 Read block 509 len 32768 20/30 Read block 119 len 32768 21/30 Read block 547 len 40960 22/30 Read block 461 len 12288 23/30 Read block 505 len 8192 24/30 Flush 25/30 Write block 54 len 20480 data: 14 14 14 16 16 26/30 Read block 423 len 16384 27/30 Write block 156 len 8192 data: 14 12 28/30 Write block 415 len 24576 data: 13 15 14 13 15 14 29/30 Write block 358 len 36864 data: 10 10 10 11 11 12 12 13 13 30/30 Write block 115 len 24576 data: 10 9 7 10 9 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 3 3 33 Done 3 27 27 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:02:54.996961843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:02:54.997160128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1810,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 146 146 146 254 254 254 F F F 2 040-059 140 140 140 243 243 243 F T T <--- 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 148 150 150 256 260 260 F T T <--- 6 120-139 149 150 150 257 260 260 F T T <--- 7 140-159 149 150 150 257 258 258 F T T <--- 8 160-179 142 142 142 245 245 245 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 149 149 149 257 257 257 F F F 14 280-299 149 149 149 257 257 257 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 140 140 140 243 243 243 F T T <--- 18 360-379 147 147 147 255 255 255 F T T <--- 19 380-399 148 148 148 256 256 256 F F F 20 400-419 149 150 150 257 260 260 F T T <--- 21 420-439 147 147 147 255 255 255 F T T <--- 22 440-459 148 150 150 256 260 260 F F F <--- 23 460-479 148 150 150 256 260 260 F F F <--- 24 480-499 146 150 150 252 258 258 F F F <--- 25 500-519 148 150 150 256 258 258 T F F <--- 26 520-539 148 148 148 256 256 256 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 150, Max flush: 260 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 146 146 146 254 254 254 F F F 2 040-059 151 151 151 261 261 261 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 151 151 151 261 261 261 F F F 6 120-139 151 151 151 261 261 261 F F F 7 140-159 151 151 151 261 261 261 F F F 8 160-179 142 142 142 245 245 245 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 149 149 149 257 257 257 F F F 14 280-299 149 149 149 257 257 257 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 151 151 151 261 261 261 F F F 18 360-379 151 151 151 261 261 261 F F F 19 380-399 148 148 148 256 256 256 F F F 20 400-419 151 151 151 261 261 261 F F F 21 420-439 151 151 151 261 261 261 F F F 22 440-459 150 150 150 260 260 260 F F F 23 460-479 150 150 150 260 260 260 F F F 24 480-499 150 150 150 258 258 258 F F F 25 500-519 150 150 150 258 258 258 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F F F Max gen: 151, Max flush: 261 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.253541777Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:09.260509613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.260563856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267224222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267483949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267508966Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267517334Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267528713Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Crucible e7e45886-ab35-44c0-959b-637976786da1 has session id: 210c9348-3034-4610-b807-603d873e2c43","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267880988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Upstairs opts: Upstairs UUID: e7e45886-ab35-44c0-959b-637976786da1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.267926133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Crucible stats registered with UUID: e7e45886-ab35-44c0-959b-637976786da1","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.26794546Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:09.267954569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268075364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268109449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268120298Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268250012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268273949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"e7e45886-ab35-44c0-959b-637976786da1 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268282527Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268291306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"0","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268300915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"1","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268310564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"2","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268320043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268329981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.26833971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:43706)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268387504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:58244)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268398233Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:47851)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.268467684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269190362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"2","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.26921051Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"0","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269221998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"1","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269536128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[0]R flush_numbers[0..12]: [257, 254, 261, 245, 241, 261, 261, 261, 245, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269559635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[0]R generation[0..12]: [149, 146, 151, 142, 138, 151, 151, 151, 142, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269571534Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269582643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[1]R flush_numbers[0..12]: [257, 254, 261, 245, 241, 261, 261, 261, 245, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269593311Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[1]R generation[0..12]: [149, 146, 151, 142, 138, 151, 151, 151, 142, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.26960383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269614898Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[2]R flush_numbers[0..12]: [257, 254, 261, 245, 241, 261, 261, 261, 245, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269625497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[2]R generation[0..12]: [149, 146, 151, 142, 138, 151, 151, 151, 142, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269636046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269649544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269659083Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269668802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.26967812Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269687469Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"":"downstairs","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269697118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269705747Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"0","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269715356Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"1","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269724885Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"client":"2","":"downstairs","session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269734503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269743572Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"e7e45886-ab35-44c0-959b-637976786da1 is now active with session: 210c9348-3034-4610-b807-603d873e2c43","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269753511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.26976199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"session_id":"210c9348-3034-4610-b807-603d873e2c43"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269771139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Activated sub_volume e7e45886-ab35-44c0-959b-637976786da1","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269778968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.269787107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.270104226Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} 01/30 Write block 554 len 16384 data: 6 9 6 7 02/30 Read block 67 len 36864 03/30 Read block 149 len 36864 04/30 Read block 297 len 24576 05/30 Read block 537 len 28672 06/30 Read block 555 len 8192 07/30 Write block 350 len 28672 data: 11 11 14 12 11 9 10 08/30 Read block 484 len 16384 09/30 Read block 545 len 28672 10/30 Write block 154 len 36864 data: 15 14 15 13 12 10 10 11 10 11/30 Read block 453 len 32768 12/30 Write block 172 len 4096 data: 19 13/30 Write block 595 len 20480 data: 10 8 6 5 4 14/30 Read block 232 len 24576 15/30 Write block 139 len 8192 data: 13 9 16/30 Read block 291 len 4096 17/30 Read block 370 len 40960 18/30 Read block 378 len 40960 19/30 Read block 276 len 24576 20/30 Write block 457 len 40960 data: 13 12 14 14 14 16 20 14 15 13 21/30 Read block 466 len 24576 22/30 Write block 474 len 4096 data: 8 23/30 Read block 290 len 28672 24/30 Write block 156 len 12288 data: 16 14 13 25/30 Read block 529 len 8192 26/30 Read block 565 len 36864 27/30 Write block 443 len 40960 data: 10 11 14 15 14 12 13 9 10 10 28/30 Write block 512 len 20480 data: 14 10 13 14 16 29/30 Write block 150 len 28672 data: 14 10 14 15 16 15 17 30/30 Write block 271 len 32768 data: 9 10 11 10 13 14 15 13 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 10 Sent Sent Sent false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 4 4 36 Done 2 26 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:09.278967712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:09.279193744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1819,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 146 146 146 254 254 254 F F F 2 040-059 151 151 151 261 261 261 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 151 151 151 261 261 261 F F F 6 120-139 151 151 151 261 261 261 F T T <--- 7 140-159 151 151 151 261 261 261 F T T <--- 8 160-179 142 142 142 245 245 245 F T T <--- 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 149 149 149 257 257 257 F T T <--- 14 280-299 149 149 149 257 257 257 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 151 151 151 261 261 261 F T T <--- 18 360-379 151 151 151 261 261 261 F F F 19 380-399 148 148 148 256 256 256 F F F 20 400-419 151 151 151 261 261 261 F F F 21 420-439 151 151 151 261 261 261 F F F 22 440-459 150 150 150 260 260 260 F T T <--- 23 460-479 150 150 150 260 260 260 F T T <--- 24 480-499 150 150 150 258 258 258 F F F 25 500-519 150 150 150 258 258 258 F T T <--- 26 520-539 148 148 148 256 256 256 F F F 27 540-559 148 148 148 256 256 256 T T T 28 560-579 133 133 133 228 228 228 F F F 29 580-599 139 139 139 242 242 242 F T T <--- Max gen: 151, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 146 146 146 254 254 254 F F F 2 040-059 151 151 151 261 261 261 F F F 3 060-079 142 142 142 245 245 245 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 151 151 151 261 261 261 F F F 6 120-139 152 152 152 262 262 262 F F F 7 140-159 152 152 152 262 262 262 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 152 152 152 262 262 262 F F F 14 280-299 149 149 149 257 257 257 F F F 15 300-319 144 144 144 249 249 249 F F F 16 320-339 138 138 138 241 241 241 F F F 17 340-359 152 152 152 262 262 262 F F F 18 360-379 151 151 151 261 261 261 F F F 19 380-399 148 148 148 256 256 256 F F F 20 400-419 151 151 151 261 261 261 F F F 21 420-439 151 151 151 261 261 261 F F F 22 440-459 152 152 152 262 262 262 F F F 23 460-479 152 152 152 262 262 262 F F F 24 480-499 150 150 150 258 258 258 F F F 25 500-519 152 152 152 262 262 262 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 152 152 152 262 262 262 F F F Max gen: 152, Max flush: 262 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.535265852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:23.542287741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.542341414Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549006099Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549261017Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549285214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549293843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549305331Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Crucible 5b5ea194-a233-40e6-aff0-a4b9ca74420c has session id: 0770577a-4af8-44c9-9030-399de7941608","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.54971074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Upstairs opts: Upstairs UUID: 5b5ea194-a233-40e6-aff0-a4b9ca74420c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549726718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Crucible stats registered with UUID: 5b5ea194-a233-40e6-aff0-a4b9ca74420c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549739686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:23.549748845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549884908Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549915474Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.549928762Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550061305Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550079673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"5b5ea194-a233-40e6-aff0-a4b9ca74420c active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.55010277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550112309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"0","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550121848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"1","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550131436Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"2","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550140985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550150834Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.55017983Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"ds_connection connected from Ok(127.0.0.1:55242)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550219765Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"ds_connection connected from Ok(127.0.0.1:40443)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550310614Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"ds_connection connected from Ok(127.0.0.1:59209)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550321742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550879861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"1","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.5508938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"2","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.550977249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"0","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551269282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[0]R flush_numbers[0..12]: [257, 254, 261, 245, 241, 261, 262, 262, 262, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551294519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[0]R generation[0..12]: [149, 146, 151, 142, 138, 151, 152, 152, 152, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551305597Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551316426Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[1]R flush_numbers[0..12]: [257, 254, 261, 245, 241, 261, 262, 262, 262, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551326915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[1]R generation[0..12]: [149, 146, 151, 142, 138, 151, 152, 152, 152, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551337413Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551348262Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[2]R flush_numbers[0..12]: [257, 254, 261, 245, 241, 261, 262, 262, 262, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551358681Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[2]R generation[0..12]: [149, 146, 151, 142, 138, 151, 152, 152, 152, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551368979Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551379668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Max found gen is 153","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551388967Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Generation requested: 154 >= found:153","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551398626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551408054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551419083Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"":"downstairs","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551428742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551437461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"0","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551447179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"1","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551456578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"client":"2","":"downstairs","session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551466227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551475236Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"5b5ea194-a233-40e6-aff0-a4b9ca74420c is now active with session: 0770577a-4af8-44c9-9030-399de7941608","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551485155Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551493734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"session_id":"0770577a-4af8-44c9-9030-399de7941608"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551502872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Activated sub_volume 5b5ea194-a233-40e6-aff0-a4b9ca74420c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551510961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.55151904Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.551813633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} 01/30 Write block 361 len 32768 data: 12 12 13 13 14 14 12 9 02/30 Write block 29 len 8192 data: 10 11 03/30 Write block 314 len 32768 data: 11 13 11 9 9 8 11 9 04/30 Read block 418 len 20480 05/30 Read block 466 len 36864 06/30 Read block 380 len 12288 07/30 Flush 08/30 Write block 564 len 12288 data: 5 6 7 09/30 Flush 10/30 Write block 528 len 32768 data: 11 10 10 11 13 15 15 15 11/30 Read block 436 len 28672 12/30 Write block 291 len 24576 data: 6 7 9 9 11 10 13/30 Read block 303 len 36864 14/30 Write block 514 len 40960 data: 14 15 17 15 12 14 17 15 15 16 15/30 Write block 276 len 36864 data: 15 16 14 14 16 16 15 17 13 16/30 Read block 128 len 20480 17/30 Read block 155 len 12288 18/30 Write block 293 len 12288 data: 10 10 12 19/30 Flush 20/30 Write block 454 len 24576 data: 15 12 13 14 13 15 21/30 Write block 341 len 32768 data: 11 12 13 13 16 18 20 17 22/30 Write block 125 len 36864 data: 19 20 19 16 18 17 20 22 17 23/30 Read block 395 len 24576 24/30 Read block 300 len 28672 25/30 Read block 328 len 4096 26/30 Write block 405 len 4096 data: 11 27/30 Write block 473 len 4096 data: 7 28/30 Write block 61 len 4096 data: 14 29/30 Write block 569 len 40960 data: 10 6 5 3 5 8 11 12 11 10 30/30 Write block 402 len 40960 data: 14 13 13 12 10 11 13 12 10 10 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:23.560365508Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:23.560529328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1828,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 146 154 154 254 263 263 F F F <--- 2 040-059 151 151 151 261 261 261 F F F 3 060-079 142 142 142 245 245 245 F T T <--- 4 080-099 138 138 138 241 241 241 F F F 5 100-119 151 151 151 261 261 261 F F F 6 120-139 152 152 152 262 262 262 F T T <--- 7 140-159 152 152 152 262 262 262 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 152 154 154 262 265 265 F F F <--- 14 280-299 149 154 154 257 265 265 F F F <--- 15 300-319 144 154 154 249 263 263 F F F <--- 16 320-339 138 154 154 241 263 263 F F F <--- 17 340-359 152 152 152 262 262 262 F T T <--- 18 360-379 151 154 154 261 263 263 T F F <--- 19 380-399 148 148 148 256 256 256 F F F 20 400-419 151 151 151 261 261 261 F T T <--- 21 420-439 151 151 151 261 261 261 F F F 22 440-459 152 152 152 262 262 262 F T T <--- 23 460-479 152 152 152 262 262 262 F T T <--- 24 480-499 150 150 150 258 258 258 F F F 25 500-519 152 154 154 262 265 265 F F F <--- 26 520-539 148 154 154 256 265 265 F F F <--- 27 540-559 152 152 152 262 262 262 F F F 28 560-579 133 154 154 228 264 264 F T T <--- 29 580-599 152 152 152 262 262 262 F F F Max gen: 154, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 151 151 151 261 261 261 F F F 3 060-079 155 155 155 266 266 266 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 151 151 151 261 261 261 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 262 262 262 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 146 146 146 254 254 254 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F F F 13 260-279 154 154 154 265 265 265 F F F 14 280-299 154 154 154 265 265 265 F F F 15 300-319 154 154 154 263 263 263 F F F 16 320-339 154 154 154 263 263 263 F F F 17 340-359 155 155 155 266 266 266 F F F 18 360-379 154 154 154 263 263 263 F F F 19 380-399 148 148 148 256 256 256 F F F 20 400-419 155 155 155 266 266 266 F F F 21 420-439 151 151 151 261 261 261 F F F 22 440-459 155 155 155 266 266 266 F F F 23 460-479 155 155 155 266 266 266 F F F 24 480-499 150 150 150 258 258 258 F F F 25 500-519 154 154 154 265 265 265 F F F 26 520-539 154 154 154 265 265 265 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 152 152 152 262 262 262 F F F Max gen: 155, Max flush: 266 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.827662293Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:37.834734786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.834788419Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.841474031Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.841734228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.841758915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.841767184Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.841778582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Crucible 0dfbed01-aa0c-4301-81f2-ba13d6dc9c41 has session id: debc4dab-72cb-436c-b216-d21322ec8caf","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842138187Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Upstairs opts: Upstairs UUID: 0dfbed01-aa0c-4301-81f2-ba13d6dc9c41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842166853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Crucible stats registered with UUID: 0dfbed01-aa0c-4301-81f2-ba13d6dc9c41","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842177492Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:37.842186671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842329592Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.8423456Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842355969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.84250694Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842523618Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"0dfbed01-aa0c-4301-81f2-ba13d6dc9c41 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842540586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842551054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"0","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842560753Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"1","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842570312Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"2","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842579761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842589679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842599628Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"ds_connection connected from Ok(127.0.0.1:35892)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842680988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"1","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54923)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842695206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"0","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43230)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.842709574Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"io task","client":"2","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843288991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"1","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843347793Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"2","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843394737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"0","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.84368915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[0]R flush_numbers[0..12]: [257, 263, 261, 266, 241, 261, 266, 262, 262, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843718226Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[0]R generation[0..12]: [149, 154, 151, 155, 138, 151, 155, 152, 152, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843729585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843743473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[1]R flush_numbers[0..12]: [257, 263, 261, 266, 241, 261, 266, 262, 262, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843754332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[1]R generation[0..12]: [149, 154, 151, 155, 138, 151, 155, 152, 152, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.84376467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843775709Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[2]R flush_numbers[0..12]: [257, 263, 261, 266, 241, 261, 266, 262, 262, 254, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843786268Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[2]R generation[0..12]: [149, 154, 151, 155, 138, 151, 155, 152, 152, 146, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843796906Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843807815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843817224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843826963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843836291Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.84384559Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"":"downstairs","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843855399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843864138Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"0","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843873657Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"1","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843885135Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"client":"2","":"downstairs","session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843894824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843903963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"0dfbed01-aa0c-4301-81f2-ba13d6dc9c41 is now active with session: debc4dab-72cb-436c-b216-d21322ec8caf","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843913852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.84392256Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"session_id":"debc4dab-72cb-436c-b216-d21322ec8caf"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843931569Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Activated sub_volume 0dfbed01-aa0c-4301-81f2-ba13d6dc9c41","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843939588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.843947707Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.844255988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} 01/30 Write block 427 len 8192 data: 14 12 02/30 Read block 466 len 24576 03/30 Read block 269 len 24576 04/30 Write block 385 len 12288 data: 10 9 12 05/30 Write block 492 len 32768 data: 10 11 7 6 6 4 6 7 06/30 Write block 395 len 8192 data: 9 7 07/30 Read block 301 len 36864 08/30 Read block 428 len 4096 09/30 Write block 42 len 24576 data: 10 9 12 10 13 13 10/30 Write block 176 len 20480 data: 15 14 9 10 12 11/30 Write block 306 len 20480 data: 13 14 15 11 11 12/30 Write block 287 len 12288 data: 13 8 8 13/30 Write block 482 len 20480 data: 11 11 12 11 10 14/30 Write block 104 len 32768 data: 11 11 11 15 16 15 14 13 15/30 Read block 252 len 20480 16/30 Flush 17/30 Write block 446 len 12288 data: 16 15 13 18/30 Read block 75 len 20480 19/30 Read block 472 len 8192 20/30 Read block 133 len 28672 21/30 Read block 157 len 8192 22/30 Read block 6 len 36864 23/30 Read block 33 len 4096 24/30 Read block 482 len 20480 25/30 Write block 314 len 24576 data: 12 14 12 10 10 9 26/30 Write block 250 len 8192 data: 9 11 27/30 Read block 133 len 28672 28/30 Write block 373 len 32768 data: 11 11 13 14 14 12 12 9 29/30 Read block 449 len 12288 30/30 Read block 528 len 24576 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:37.852307197Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:37.852469496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1837,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 151 156 156 261 267 267 F F F <--- 3 060-079 155 155 155 266 266 266 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 151 156 156 261 267 267 F F F <--- 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 262 262 262 F F F 8 160-179 152 156 156 262 267 267 F F F <--- 9 180-199 146 156 156 254 267 267 F F F <--- 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 148 148 148 256 256 256 F T T <--- 13 260-279 154 154 154 265 265 265 F F F 14 280-299 154 156 156 265 267 267 F F F <--- 15 300-319 154 156 156 263 267 267 F T T <--- 16 320-339 154 154 154 263 263 263 F F F 17 340-359 155 155 155 266 266 266 F F F 18 360-379 154 154 154 263 263 263 F T T <--- 19 380-399 148 156 156 256 267 267 F T T <--- 20 400-419 155 155 155 266 266 266 F F F 21 420-439 151 156 156 261 267 267 F F F <--- 22 440-459 155 155 155 266 266 266 F T T <--- 23 460-479 155 155 155 266 266 266 F F F 24 480-499 150 156 156 258 267 267 F F F <--- 25 500-519 154 154 154 265 265 265 F F F 26 520-539 154 154 154 265 265 265 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 152 152 152 262 262 262 F F F Max gen: 156, Max flush: 267 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 156 156 156 267 267 267 F F F 3 060-079 155 155 155 266 266 266 F F F 4 080-099 138 138 138 241 241 241 F F F 5 100-119 156 156 156 267 267 267 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 262 262 262 F F F 8 160-179 156 156 156 267 267 267 F F F 9 180-199 156 156 156 267 267 267 F F F 10 200-219 145 145 145 250 250 250 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 157 157 157 268 268 268 F F F 13 260-279 154 154 154 265 265 265 F F F 14 280-299 156 156 156 267 267 267 F F F 15 300-319 157 157 157 268 268 268 F F F 16 320-339 154 154 154 263 263 263 F F F 17 340-359 155 155 155 266 266 266 F F F 18 360-379 157 157 157 268 268 268 F F F 19 380-399 157 157 157 268 268 268 F F F 20 400-419 155 155 155 266 266 266 F F F 21 420-439 156 156 156 267 267 267 F F F 22 440-459 157 157 157 268 268 268 F F F 23 460-479 155 155 155 266 266 266 F F F 24 480-499 156 156 156 267 267 267 F F F 25 500-519 154 154 154 265 265 265 F F F 26 520-539 154 154 154 265 265 265 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 152 152 152 262 262 262 F F F Max gen: 157, Max flush: 268 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.109579273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.117586317Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.117875411Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.117899338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.117907986Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.117919355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Crucible d79c78c7-c6a3-4a07-8e73-e62a3000f85d has session id: 61889649-72dc-4f64-8bdd-f2409c374927","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118286468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Upstairs opts: Upstairs UUID: d79c78c7-c6a3-4a07-8e73-e62a3000f85d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.11835094Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Crucible stats registered with UUID: d79c78c7-c6a3-4a07-8e73-e62a3000f85d","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118362339Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:52.118371558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118524168Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118540326Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118550095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118703146Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118720173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"d79c78c7-c6a3-4a07-8e73-e62a3000f85d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.11874352Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118752819Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"0","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118762258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"1","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118771857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"2","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118781376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118791114Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118800993Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"ds_connection connected from Ok(127.0.0.1:55816)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.118880653Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"ds_connection connected from Ok(127.0.0.1:61106)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119011407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"ds_connection connected from Ok(127.0.0.1:55920)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119048772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119504484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"2","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119544639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"1","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119704089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"0","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119963226Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[0]R flush_numbers[0..12]: [257, 263, 267, 266, 241, 267, 266, 262, 267, 267, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.119989252Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[0]R generation[0..12]: [149, 154, 156, 155, 138, 156, 155, 152, 156, 156, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.12001106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120022328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[1]R flush_numbers[0..12]: [257, 263, 267, 266, 241, 267, 266, 262, 267, 267, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120033007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[1]R generation[0..12]: [149, 154, 156, 155, 138, 156, 155, 152, 156, 156, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120043536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120054464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[2]R flush_numbers[0..12]: [257, 263, 267, 266, 241, 267, 266, 262, 267, 267, 250, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120065053Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[2]R generation[0..12]: [149, 154, 156, 155, 138, 156, 155, 152, 156, 156, 145, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120077771Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.12008868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120097999Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120107537Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120116856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120126225Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"":"downstairs","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120136134Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120145123Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"0","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120154961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"1","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.12016481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"client":"2","":"downstairs","session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120174699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120183668Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"d79c78c7-c6a3-4a07-8e73-e62a3000f85d is now active with session: 61889649-72dc-4f64-8bdd-f2409c374927","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120193577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120202295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"session_id":"61889649-72dc-4f64-8bdd-f2409c374927"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120211284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Activated sub_volume d79c78c7-c6a3-4a07-8e73-e62a3000f85d","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120219353Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120228782Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.120517466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"task":"crutest"} 01/30 Write block 311 len 12288 data: 12 11 9 02/30 Write block 355 len 28672 data: 10 11 11 11 11 11 13 03/30 Read block 3 len 40960 04/30 Write block 214 len 24576 data: 17 14 13 13 16 17 05/30 Read block 532 len 16384 06/30 Read block 528 len 40960 07/30 Read block 436 len 16384 08/30 Write block 525 len 36864 data: 17 14 13 12 11 11 12 14 16 09/30 Write block 49 len 32768 data: 17 15 16 12 13 15 15 15 10/30 Write block 65 len 40960 data: 10 9 10 9 8 8 13 14 12 15 11/30 Read block 1 len 20480 12/30 Read block 113 len 4096 13/30 Write block 429 len 16384 data: 13 9 12 12 14/30 Flush 15/30 Write block 435 len 8192 data: 10 10 16/30 Write block 98 len 12288 data: 13 13 12 17/30 Read block 240 len 36864 18/30 Read block 479 len 12288 19/30 Write block 163 len 4096 data: 13 20/30 Write block 408 len 4096 data: 14 21/30 Read block 90 len 36864 22/30 Write block 405 len 20480 data: 13 11 12 15 13 23/30 Flush 24/30 Write block 402 len 20480 data: 15 14 14 14 12 25/30 Read block 46 len 28672 26/30 Write block 249 len 16384 data: 15 10 12 12 27/30 Write block 299 len 4096 data: 7 28/30 Write block 322 len 16384 data: 9 9 7 9 29/30 Write block 414 len 4096 data: 14 30/30 Read block 205 len 24576 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:03:52.128411844Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:03:52.128595531Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1846,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 158 156 158 269 267 269 F F F <--- 3 060-079 158 155 158 269 266 269 F F F <--- 4 080-099 158 138 158 270 241 270 F F F <--- 5 100-119 158 156 158 270 267 270 F F F <--- 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 262 262 262 F F F 8 160-179 158 156 158 270 267 270 F F F <--- 9 180-199 156 156 156 267 267 267 F F F 10 200-219 158 145 158 269 250 269 F F F <--- 11 220-239 149 149 149 257 257 257 F F F 12 240-259 157 157 157 268 268 268 T F T <--- 13 260-279 154 154 154 265 265 265 F F F 14 280-299 156 156 156 267 267 267 T F T <--- 15 300-319 158 157 158 269 268 269 F T F <--- 16 320-339 154 154 154 263 263 263 T F T <--- 17 340-359 158 155 158 269 266 269 F F F <--- 18 360-379 158 157 158 269 268 269 F F F <--- 19 380-399 157 157 157 268 268 268 F F F 20 400-419 158 155 158 270 266 270 T F T <--- 21 420-439 158 156 158 270 267 270 F F F <--- 22 440-459 157 157 157 268 268 268 F F F 23 460-479 155 155 155 266 266 266 F F F 24 480-499 156 156 156 267 267 267 F F F 25 500-519 154 154 154 265 265 265 F F F 26 520-539 158 154 158 269 265 269 F F F <--- 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 152 152 152 262 262 262 F F F Max gen: 158, Max flush: 270 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 158 158 158 269 269 269 F F F 3 060-079 158 158 158 269 269 269 F F F 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 262 262 262 F F F 8 160-179 158 158 158 270 270 270 F F F 9 180-199 156 156 156 267 267 267 F F F 10 200-219 158 158 158 269 269 269 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 159 159 159 271 271 271 F F F 13 260-279 154 154 154 265 265 265 F F F 14 280-299 159 159 159 271 271 271 F F F 15 300-319 158 158 158 269 269 269 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 158 158 158 269 269 269 F F F 18 360-379 158 158 158 269 269 269 F F F 19 380-399 157 157 157 268 268 268 F F F 20 400-419 159 159 159 271 271 271 F F F 21 420-439 158 158 158 270 270 270 F F F 22 440-459 157 157 157 268 268 268 F F F 23 460-479 155 155 155 266 266 266 F F F 24 480-499 156 156 156 267 267 267 F F F 25 500-519 154 154 154 265 265 265 F F F 26 520-539 158 158 158 269 269 269 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 152 152 152 262 262 262 F F F Max gen: 159, Max flush: 271 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.414591274Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.422566593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.422850787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.422877084Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.422888872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.422900431Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Crucible e6773e55-2c3d-48ac-b6f1-e2238917b141 has session id: 8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423271773Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Upstairs opts: Upstairs UUID: e6773e55-2c3d-48ac-b6f1-e2238917b141, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423291081Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Crucible stats registered with UUID: e6773e55-2c3d-48ac-b6f1-e2238917b141","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.42330095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:06.423310319Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423451931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423468149Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423480467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423624659Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423641147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"e6773e55-2c3d-48ac-b6f1-e2238917b141 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423653295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423662064Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"0","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423671523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"1","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423680992Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"2","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.42369065Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423721556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423743994Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57498)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423779009Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49117)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423803276Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56605)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.423814405Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.424464702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"1","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.42448182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"0","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.424674076Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"2","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.424963429Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[0]R flush_numbers[0..12]: [257, 263, 269, 269, 270, 270, 266, 262, 270, 267, 269, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.424990935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[0]R generation[0..12]: [149, 154, 158, 158, 158, 158, 155, 152, 158, 156, 158, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425002054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425013143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[1]R flush_numbers[0..12]: [257, 263, 269, 269, 270, 270, 266, 262, 270, 267, 269, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425023621Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[1]R generation[0..12]: [149, 154, 158, 158, 158, 158, 155, 152, 158, 156, 158, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.42503409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425044909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[2]R flush_numbers[0..12]: [257, 263, 269, 269, 270, 270, 266, 262, 270, 267, 269, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425055417Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[2]R generation[0..12]: [149, 154, 158, 158, 158, 158, 155, 152, 158, 156, 158, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425065876Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425076665Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425085923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425095672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425105011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.42511607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"":"downstairs","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425125678Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425134317Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"0","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425144046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"1","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425153465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"client":"2","":"downstairs","session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425163174Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425172222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"e6773e55-2c3d-48ac-b6f1-e2238917b141 is now active with session: 8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425182021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.42519078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"session_id":"8b516725-5a14-4bb6-9ebe-4d3d0f0bc39c"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425199919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Activated sub_volume e6773e55-2c3d-48ac-b6f1-e2238917b141","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425207918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Wait for a query_work_queue command to finish before sending IO","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425221086Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.425460946Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"task":"crutest"} 01/30 Flush 02/30 Write block 262 len 4096 data: 11 03/30 Read block 68 len 8192 04/30 Read block 483 len 4096 05/30 Write block 377 len 32768 data: 15 13 13 10 9 10 10 11 06/30 Read block 426 len 20480 07/30 Write block 493 len 36864 data: 12 8 7 7 5 7 8 6 7 08/30 Read block 407 len 8192 09/30 Write block 348 len 40960 data: 18 13 12 12 15 13 12 11 12 12 10/30 Write block 262 len 20480 data: 12 9 11 9 9 11/30 Flush 12/30 Write block 247 len 4096 data: 14 13/30 Write block 250 len 36864 data: 11 13 13 11 10 10 13 9 12 14/30 Read block 464 len 20480 15/30 Read block 36 len 16384 16/30 Flush 17/30 Flush 18/30 Write block 145 len 28672 data: 14 17 16 18 18 15 11 19/30 Read block 589 len 4096 20/30 Write block 309 len 12288 data: 12 12 13 21/30 Read block 590 len 40960 22/30 Read block 65 len 8192 23/30 Write block 59 len 24576 data: 15 16 15 12 11 10 24/30 Read block 556 len 24576 25/30 Read block 237 len 12288 26/30 Write block 382 len 32768 data: 11 11 12 11 10 13 12 11 27/30 Write block 594 len 20480 data: 10 11 9 7 6 28/30 Write block 189 len 36864 data: 15 12 11 14 16 13 12 10 8 29/30 Read block 551 len 28672 30/30 Read block 4 len 32768 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 1 Done Done Sent false 1002 Acked Read 2 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:06.434060795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:06.434250461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1855,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 158 158 158 269 269 269 T T F <--- 3 060-079 158 158 158 269 269 269 T T F <--- 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 152 152 152 262 262 262 T T F <--- 8 160-179 158 158 158 270 270 270 F F F 9 180-199 156 156 156 267 267 267 T T F <--- 10 200-219 158 158 158 269 269 269 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 160 160 159 274 274 271 F F F <--- 13 260-279 160 160 154 273 273 265 F F F <--- 14 280-299 159 159 159 271 271 271 F F F 15 300-319 158 158 158 269 269 269 T T F <--- 16 320-339 159 159 159 271 271 271 F F F 17 340-359 160 160 158 273 273 269 F F F <--- 18 360-379 160 160 158 273 273 269 F F F <--- 19 380-399 160 160 157 273 273 268 T T F <--- 20 400-419 159 159 159 271 271 271 F F F 21 420-439 158 158 158 270 270 270 F F F 22 440-459 157 157 157 268 268 268 F F F 23 460-479 155 155 155 266 266 266 F F F 24 480-499 160 160 156 273 273 267 F F F <--- 25 500-519 160 160 154 273 273 265 F F F <--- 26 520-539 158 158 158 269 269 269 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 152 152 152 262 262 262 T T F <--- Max gen: 160, Max flush: 274 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 161 161 161 275 275 275 F F F 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 161 161 161 275 275 275 F F F 8 160-179 158 158 158 270 270 270 F F F 9 180-199 161 161 161 275 275 275 F F F 10 200-219 158 158 158 269 269 269 F F F 11 220-239 149 149 149 257 257 257 F F F 12 240-259 160 160 160 274 274 274 F F F 13 260-279 160 160 160 273 273 273 F F F 14 280-299 159 159 159 271 271 271 F F F 15 300-319 161 161 161 275 275 275 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 161 161 161 275 275 275 F F F 20 400-419 159 159 159 271 271 271 F F F 21 420-439 158 158 158 270 270 270 F F F 22 440-459 157 157 157 268 268 268 F F F 23 460-479 155 155 155 266 266 266 F F F 24 480-499 160 160 160 273 273 273 F F F 25 500-519 160 160 160 273 273 273 F F F 26 520-539 158 158 158 269 269 269 F F F 27 540-559 152 152 152 262 262 262 F F F 28 560-579 155 155 155 266 266 266 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 161, Max flush: 275 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.701329633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709213033Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709514585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709539382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709550391Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709561809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Crucible 0c45b2c0-6951-4e41-800f-36dea77b1c08 has session id: 1a330e4d-4f67-49f1-be9b-ae0f6d593c7f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709925163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Upstairs opts: Upstairs UUID: 0c45b2c0-6951-4e41-800f-36dea77b1c08, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709955109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Crucible stats registered with UUID: 0c45b2c0-6951-4e41-800f-36dea77b1c08","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.709969317Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:20.709978506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.71010484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710121738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710131807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710294806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710311504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"0c45b2c0-6951-4e41-800f-36dea77b1c08 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710327952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710336701Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"0","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.71034635Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"1","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710357988Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"2","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710367637Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710377506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710387424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57381)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710575741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50774)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710597338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45846)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.710608096Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711112303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"0","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711127141Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"1","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711186343Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"2","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.7114444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[0]R flush_numbers[0..12]: [257, 263, 275, 275, 270, 270, 266, 275, 270, 275, 269, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711463598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[0]R generation[0..12]: [149, 154, 161, 161, 158, 158, 155, 161, 158, 161, 158, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711474247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711485125Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[1]R flush_numbers[0..12]: [257, 263, 275, 275, 270, 270, 266, 275, 270, 275, 269, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711495734Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[1]R generation[0..12]: [149, 154, 161, 161, 158, 158, 155, 161, 158, 161, 158, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711508902Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711519721Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[2]R flush_numbers[0..12]: [257, 263, 275, 275, 270, 270, 266, 275, 270, 275, 269, 257]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.71153016Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[2]R generation[0..12]: [149, 154, 161, 161, 158, 158, 155, 161, 158, 161, 158, 149]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711540878Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711551717Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711561186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711570924Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711580303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711589702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"":"downstairs","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711599321Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.71160798Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"0","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711617838Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"1","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711627277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"client":"2","":"downstairs","session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711637106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711646185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"0c45b2c0-6951-4e41-800f-36dea77b1c08 is now active with session: 1a330e4d-4f67-49f1-be9b-ae0f6d593c7f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711657553Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711666242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"session_id":"1a330e4d-4f67-49f1-be9b-ae0f6d593c7f"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711675301Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Activated sub_volume 0c45b2c0-6951-4e41-800f-36dea77b1c08","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.71168324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.711691259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.71200214Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"task":"crutest"} 01/30 Write block 301 len 36864 data: 9 9 12 13 12 14 15 16 13 02/30 Flush 03/30 Flush 04/30 Write block 564 len 36864 data: 6 7 8 8 9 11 7 6 4 05/30 Write block 60 len 8192 data: 17 16 06/30 Write block 323 len 8192 data: 10 8 07/30 Write block 423 len 32768 data: 16 16 17 15 15 13 14 10 08/30 Write block 230 len 24576 data: 8 10 8 8 8 10 09/30 Read block 488 len 24576 10/30 Write block 215 len 32768 data: 15 14 14 17 18 20 21 17 11/30 Read block 568 len 12288 12/30 Read block 55 len 20480 13/30 Read block 173 len 24576 14/30 Read block 23 len 8192 15/30 Read block 565 len 20480 16/30 Write block 303 len 12288 data: 13 14 13 17/30 Write block 446 len 20480 data: 17 16 14 14 10 18/30 Read block 464 len 4096 19/30 Write block 388 len 24576 data: 13 12 15 14 15 13 20/30 Write block 294 len 40960 data: 11 13 11 10 9 8 9 10 10 14 21/30 Read block 296 len 32768 22/30 Write block 474 len 36864 data: 9 8 8 9 8 11 11 10 12 23/30 Write block 410 len 24576 data: 11 11 13 13 15 14 24/30 Write block 172 len 36864 data: 20 18 17 18 16 15 10 11 13 25/30 Flush 26/30 Write block 9 len 12288 data: 5 9 14 27/30 Write block 318 len 36864 data: 11 10 12 10 10 11 9 10 8 28/30 Write block 149 len 28672 data: 19 16 12 15 16 17 16 29/30 Write block 547 len 20480 data: 11 10 11 12 10 30/30 Read block 67 len 20480 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:20.720221057Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:20.720396455Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1864,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 257 257 257 T T F <--- 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 162 162 161 278 278 275 F F F <--- 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 161 161 161 275 275 275 T T F <--- 8 160-179 162 162 158 278 278 270 F F F <--- 9 180-199 162 162 161 278 278 275 F F F <--- 10 200-219 162 162 158 278 278 269 F F F <--- 11 220-239 162 162 149 278 278 257 F F F <--- 12 240-259 160 160 160 274 274 274 F F F 13 260-279 160 160 160 273 273 273 F F F 14 280-299 162 162 159 278 278 271 F F F <--- 15 300-319 162 162 161 278 278 275 T T F <--- 16 320-339 162 162 159 278 278 271 T T F <--- 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 162 162 161 278 278 275 F F F <--- 20 400-419 162 162 159 278 278 271 F F F <--- 21 420-439 162 162 158 278 278 270 F F F <--- 22 440-459 162 162 157 278 278 268 F F F <--- 23 460-479 162 162 155 278 278 266 F F F <--- 24 480-499 162 162 160 278 278 273 F F F <--- 25 500-519 160 160 160 273 273 273 F F F 26 520-539 158 158 158 269 269 269 F F F 27 540-559 152 152 152 262 262 262 T T F <--- 28 560-579 162 162 155 278 278 266 F F F <--- 29 580-599 161 161 161 275 275 275 F F F Max gen: 162, Max flush: 278 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 279 279 279 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 162 162 162 278 278 278 F F F 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 155 155 155 266 266 266 F F F 7 140-159 163 163 163 279 279 279 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 162 162 162 278 278 278 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 162 162 162 278 278 278 F F F 12 240-259 160 160 160 274 274 274 F F F 13 260-279 160 160 160 273 273 273 F F F 14 280-299 162 162 162 278 278 278 F F F 15 300-319 163 163 163 279 279 279 F F F 16 320-339 163 163 163 279 279 279 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 162 162 162 278 278 278 F F F 20 400-419 162 162 162 278 278 278 F F F 21 420-439 162 162 162 278 278 278 F F F 22 440-459 162 162 162 278 278 278 F F F 23 460-479 162 162 162 278 278 278 F F F 24 480-499 162 162 162 278 278 278 F F F 25 500-519 160 160 160 273 273 273 F F F 26 520-539 158 158 158 269 269 269 F F F 27 540-559 163 163 163 279 279 279 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 163, Max flush: 279 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.988642819Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.996612578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.996898162Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.996921699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.996930458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.996941956Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Crucible e28e056a-2fe9-4575-bf12-6969c9efc5b1 has session id: fc1d5285-1694-49b1-846f-6984a0fa80ba","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997279173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Upstairs opts: Upstairs UUID: e28e056a-2fe9-4575-bf12-6969c9efc5b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.99730757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Crucible stats registered with UUID: e28e056a-2fe9-4575-bf12-6969c9efc5b1","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997321098Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:34.997330537Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997467359Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997497586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997508324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997679153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.99769699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"e28e056a-2fe9-4575-bf12-6969c9efc5b1 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997710299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997719018Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"0","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997728606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"1","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997738115Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"2","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997747714Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997759042Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997828424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:60110)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997847261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:53575)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.997928681Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:35793)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.99793982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998502628Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"2","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998514277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"0","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998524855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"1","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998896958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[0]R flush_numbers[0..12]: [279, 263, 275, 278, 270, 270, 266, 279, 278, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998930054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[0]R generation[0..12]: [163, 154, 161, 162, 158, 158, 155, 163, 162, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998941123Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998952001Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[1]R flush_numbers[0..12]: [279, 263, 275, 278, 270, 270, 266, 279, 278, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.99896251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[1]R generation[0..12]: [163, 154, 161, 162, 158, 158, 155, 163, 162, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998972799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998983577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[2]R flush_numbers[0..12]: [279, 263, 275, 278, 270, 270, 266, 279, 278, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.998994206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[2]R generation[0..12]: [163, 154, 161, 162, 158, 158, 155, 163, 162, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999012603Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999024882Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999034021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.99904364Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999052828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999062077Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"":"downstairs","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999076425Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999085244Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"0","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999094963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"1","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999104702Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"client":"2","":"downstairs","session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999114721Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999123809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"e28e056a-2fe9-4575-bf12-6969c9efc5b1 is now active with session: fc1d5285-1694-49b1-846f-6984a0fa80ba","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999133628Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999142217Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"session_id":"fc1d5285-1694-49b1-846f-6984a0fa80ba"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999151336Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Activated sub_volume e28e056a-2fe9-4575-bf12-6969c9efc5b1","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999159195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999167074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:34.999483064Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"task":"crutest"} 01/30 Write block 430 len 20480 data: 11 13 13 11 11 02/30 Read block 216 len 20480 03/30 Write block 301 len 36864 data: 11 11 15 15 14 15 16 17 14 04/30 Read block 578 len 32768 05/30 Read block 144 len 24576 06/30 Read block 59 len 40960 07/30 Read block 593 len 16384 08/30 Read block 53 len 36864 09/30 Write block 298 len 20480 data: 10 9 10 12 12 10/30 Read block 223 len 28672 11/30 Write block 129 len 36864 data: 19 18 21 23 18 18 15 14 12 12/30 Write block 305 len 12288 data: 15 16 17 13/30 Read block 182 len 40960 14/30 Write block 337 len 4096 data: 13 15/30 Read block 183 len 12288 16/30 Read block 219 len 40960 17/30 Read block 129 len 16384 18/30 Write block 249 len 8192 data: 16 12 19/30 Read block 490 len 32768 20/30 Read block 88 len 40960 21/30 Write block 488 len 16384 data: 8 6 5 7 22/30 Read block 233 len 32768 23/30 Read block 34 len 36864 24/30 Write block 285 len 32768 data: 14 16 14 9 9 6 7 8 25/30 Read block 409 len 12288 26/30 Read block 494 len 20480 27/30 Read block 287 len 20480 28/30 Read block 466 len 12288 29/30 Read block 398 len 4096 30/30 Write block 459 len 20480 data: 16 15 15 17 21 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:35.008665019Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:35.008865314Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1873,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 279 279 279 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 162 162 162 278 278 278 F F F 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 155 155 155 266 266 266 T F T <--- 7 140-159 163 163 163 279 279 279 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 162 162 162 278 278 278 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 162 162 162 278 278 278 F F F 12 240-259 160 160 160 274 274 274 T F T <--- 13 260-279 160 160 160 273 273 273 F F F 14 280-299 162 162 162 278 278 278 T F T <--- 15 300-319 163 163 163 279 279 279 T F T <--- 16 320-339 163 163 163 279 279 279 T F T <--- 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 162 162 162 278 278 278 F F F 20 400-419 162 162 162 278 278 278 F F F 21 420-439 162 162 162 278 278 278 T F T <--- 22 440-459 162 162 162 278 278 278 T F T <--- 23 460-479 162 162 162 278 278 278 T F T <--- 24 480-499 162 162 162 278 278 278 T F T <--- 25 500-519 160 160 160 273 273 273 F F F 26 520-539 158 158 158 269 269 269 F F F 27 540-559 163 163 163 279 279 279 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 163, Max flush: 279 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 279 279 279 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 162 162 162 278 278 278 F F F 4 080-099 158 158 158 270 270 270 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 164 164 164 280 280 280 F F F 7 140-159 163 163 163 279 279 279 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 162 162 162 278 278 278 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 162 162 162 278 278 278 F F F 12 240-259 164 164 164 280 280 280 F F F 13 260-279 160 160 160 273 273 273 F F F 14 280-299 164 164 164 280 280 280 F F F 15 300-319 164 164 164 280 280 280 F F F 16 320-339 164 164 164 280 280 280 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 162 162 162 278 278 278 F F F 20 400-419 162 162 162 278 278 278 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 164 164 164 280 280 280 F F F 23 460-479 164 164 164 280 280 280 F F F 24 480-499 164 164 164 280 280 280 F F F 25 500-519 160 160 160 273 273 273 F F F 26 520-539 158 158 158 269 269 269 F F F 27 540-559 163 163 163 279 279 279 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 164, Max flush: 280 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.257329387Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.26526924Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.265551184Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.265576551Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.26558497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.265596408Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Crucible b7aaa2bb-e74f-476b-81e1-01dedc552242 has session id: 567be8fd-2e4f-4947-9830-7b9ce72debcb","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.265983899Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Upstairs opts: Upstairs UUID: b7aaa2bb-e74f-476b-81e1-01dedc552242, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266015015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Crucible stats registered with UUID: b7aaa2bb-e74f-476b-81e1-01dedc552242","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266035683Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:49.266045971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266177615Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266208541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266219429Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266317407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266343964Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"b7aaa2bb-e74f-476b-81e1-01dedc552242 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266357082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266365881Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"0","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.2663754Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"1","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266385228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"2","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266394957Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266404856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266414545Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:47740)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266460089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58549)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266470798Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37230)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.266499894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267279015Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"0","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267303482Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"1","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.26731468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"2","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267597895Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[0]R flush_numbers[0..12]: [279, 263, 275, 278, 270, 270, 280, 279, 278, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267616122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[0]R generation[0..12]: [163, 154, 161, 162, 158, 158, 164, 163, 162, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267626891Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267637839Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[1]R flush_numbers[0..12]: [279, 263, 275, 278, 270, 270, 280, 279, 278, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267648418Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[1]R generation[0..12]: [163, 154, 161, 162, 158, 158, 164, 163, 162, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267658837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267669625Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[2]R flush_numbers[0..12]: [279, 263, 275, 278, 270, 270, 280, 279, 278, 278, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267680084Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[2]R generation[0..12]: [163, 154, 161, 162, 158, 158, 164, 163, 162, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267690673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267701361Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Max found gen is 165","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.26771063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Generation requested: 166 >= found:165","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267720409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267729718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267739027Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"":"downstairs","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267748875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267759064Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"0","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267768933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"1","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267778742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"client":"2","":"downstairs","session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.26778874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267797919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"b7aaa2bb-e74f-476b-81e1-01dedc552242 is now active with session: 567be8fd-2e4f-4947-9830-7b9ce72debcb","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267807788Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267816557Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"session_id":"567be8fd-2e4f-4947-9830-7b9ce72debcb"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267825806Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Activated sub_volume b7aaa2bb-e74f-476b-81e1-01dedc552242","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267833715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.267841804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.268163523Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"task":"crutest"} 01/30 Write block 80 len 40960 data: 14 10 9 8 12 14 15 14 20 17 02/30 Flush 03/30 Write block 185 len 16384 data: 14 15 15 14 04/30 Read block 398 len 16384 05/30 Read block 88 len 32768 06/30 Read block 566 len 20480 07/30 Write block 149 len 16384 data: 20 17 13 16 08/30 Flush 09/30 Write block 136 len 12288 data: 15 13 11 10/30 Read block 491 len 32768 11/30 Flush 12/30 Write block 7 len 24576 data: 4 6 6 10 15 16 13/30 Write block 454 len 28672 data: 16 13 14 15 14 17 16 14/30 Write block 288 len 16384 data: 10 10 7 8 15/30 Write block 518 len 16384 data: 13 15 18 16 16/30 Read block 370 len 32768 17/30 Read block 315 len 36864 18/30 Read block 10 len 4096 19/30 Flush 20/30 Write block 311 len 12288 data: 14 12 10 21/30 Write block 390 len 32768 data: 16 15 16 14 12 10 8 8 22/30 Read block 132 len 32768 23/30 Read block 373 len 32768 24/30 Read block 577 len 16384 25/30 Read block 582 len 20480 26/30 Read block 531 len 16384 27/30 Write block 409 len 40960 data: 14 12 12 14 14 16 15 16 15 14 28/30 Write block 84 len 4096 data: 13 29/30 Read block 369 len 20480 30/30 Read block 170 len 40960 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 29 1 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:04:49.277406081Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:04:49.277588188Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1882,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 163 166 284 279 284 F F F <--- 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 162 162 162 278 278 278 F F F 4 080-099 166 158 166 281 270 281 T T T <--- 5 100-119 158 158 158 270 270 270 F F F 6 120-139 166 164 166 283 280 283 F F F <--- 7 140-159 166 163 166 282 279 282 F F F <--- 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 162 166 282 278 282 F F F <--- 10 200-219 162 162 162 278 278 278 F F F 11 220-239 162 162 162 278 278 278 F F F 12 240-259 164 164 164 280 280 280 F F F 13 260-279 160 160 160 273 273 273 F F F 14 280-299 166 164 166 284 280 284 F F F <--- 15 300-319 164 164 164 280 280 280 T F T <--- 16 320-339 164 164 164 280 280 280 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 162 162 162 278 278 278 T F T <--- 20 400-419 162 162 162 278 278 278 T F T <--- 21 420-439 164 164 164 280 280 280 F F F 22 440-459 166 164 166 284 280 284 F F F <--- 23 460-479 166 164 166 284 280 284 F F F <--- 24 480-499 164 164 164 280 280 280 F F F 25 500-519 166 160 166 284 273 284 F F F <--- 26 520-539 166 158 166 284 269 284 F F F <--- 27 540-559 163 163 163 279 279 279 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 166, Max flush: 284 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 284 284 284 F F F 1 020-039 154 154 154 263 263 263 F F F 2 040-059 161 161 161 275 275 275 F F F 3 060-079 162 162 162 278 278 278 F F F 4 080-099 167 167 167 285 285 285 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 166 166 166 282 282 282 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 162 162 162 278 278 278 F F F 12 240-259 164 164 164 280 280 280 F F F 13 260-279 160 160 160 273 273 273 F F F 14 280-299 166 166 166 284 284 284 F F F 15 300-319 167 167 167 285 285 285 F F F 16 320-339 164 164 164 280 280 280 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 160 160 160 273 273 273 F F F 19 380-399 167 167 167 285 285 285 F F F 20 400-419 167 167 167 285 285 285 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 166 166 166 284 284 284 F F F 23 460-479 166 166 166 284 284 284 F F F 24 480-499 164 164 164 280 280 280 F F F 25 500-519 166 166 166 284 284 284 F F F 26 520-539 166 166 166 284 284 284 F F F 27 540-559 163 163 163 279 279 279 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 167, Max flush: 285 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.535420422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.543367744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.543650689Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.543677895Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.543686264Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.543697793Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Crucible a08caa38-c281-438b-a1d1-b2d57bc46d62 has session id: 4f97f828-e759-41ae-87c1-49b6a041ba63","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544042789Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Upstairs opts: Upstairs UUID: a08caa38-c281-438b-a1d1-b2d57bc46d62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544071465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Crucible stats registered with UUID: a08caa38-c281-438b-a1d1-b2d57bc46d62","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544082604Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:03.544091963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544210168Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544226406Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544236354Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544377846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544394284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"a08caa38-c281-438b-a1d1-b2d57bc46d62 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544408832Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544417561Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"0","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.54442707Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"1","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544436879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"2","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544446358Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544456316Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544468465Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"ds_connection connected from Ok(127.0.0.1:48666)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544532857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"ds_connection connected from Ok(127.0.0.1:53377)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544557874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"ds_connection connected from Ok(127.0.0.1:53053)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.544569032Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545238387Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"2","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545271253Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"1","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545366671Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"0","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545634277Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[0]R flush_numbers[0..12]: [284, 263, 275, 278, 285, 270, 283, 282, 278, 282, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545659284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[0]R generation[0..12]: [166, 154, 161, 162, 167, 158, 166, 166, 162, 166, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545670392Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545681291Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[1]R flush_numbers[0..12]: [284, 263, 275, 278, 285, 270, 283, 282, 278, 282, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.54569188Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[1]R generation[0..12]: [166, 154, 161, 162, 167, 158, 166, 166, 162, 166, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545702428Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545715867Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[2]R flush_numbers[0..12]: [284, 263, 275, 278, 285, 270, 283, 282, 278, 282, 278, 278]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545726385Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[2]R generation[0..12]: [166, 154, 161, 162, 167, 158, 166, 166, 162, 166, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545736824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545747623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545757011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.54576662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545775879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545785288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"":"downstairs","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545794937Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545803726Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"0","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545813374Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"1","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545823093Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"client":"2","":"downstairs","session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545832882Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545841961Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"a08caa38-c281-438b-a1d1-b2d57bc46d62 is now active with session: 4f97f828-e759-41ae-87c1-49b6a041ba63","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545851969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545860738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"session_id":"4f97f828-e759-41ae-87c1-49b6a041ba63"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545871397Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Activated sub_volume a08caa38-c281-438b-a1d1-b2d57bc46d62","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545879366Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.545887355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.546191836Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"task":"crutest"} 01/30 Write block 28 len 32768 data: 10 11 12 11 7 7 7 6 02/30 Read block 297 len 4096 03/30 Read block 112 len 36864 04/30 Write block 397 len 12288 data: 9 7 8 05/30 Read block 375 len 20480 06/30 Write block 372 len 40960 data: 12 12 12 14 15 16 14 14 11 10 07/30 Read block 416 len 4096 08/30 Read block 426 len 32768 09/30 Read block 245 len 4096 10/30 Write block 540 len 12288 data: 20 18 14 11/30 Write block 228 len 40960 data: 7 7 9 11 9 9 9 11 12 11 12/30 Write block 50 len 32768 data: 16 17 13 14 16 16 16 17 13/30 Read block 296 len 24576 14/30 Write block 532 len 28672 data: 15 17 16 16 16 13 15 15/30 Read block 310 len 8192 16/30 Read block 81 len 20480 17/30 Read block 22 len 28672 18/30 Flush 19/30 Write block 148 len 32768 data: 19 21 18 14 17 17 18 17 20/30 Read block 350 len 28672 21/30 Write block 93 len 20480 data: 9 8 10 13 13 22/30 Read block 381 len 40960 23/30 Read block 356 len 24576 24/30 Read block 502 len 4096 25/30 Read block 209 len 12288 26/30 Read block 457 len 20480 27/30 Read block 526 len 36864 28/30 Write block 316 len 24576 data: 13 11 12 11 13 11 29/30 Write block 269 len 24576 data: 9 9 10 11 12 11 30/30 Read block 222 len 4096 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:03.555532282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:03.5557024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1891,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 284 284 284 F F F 1 020-039 168 154 168 286 263 286 F T F <--- 2 040-059 168 161 168 286 275 286 F F F <--- 3 060-079 162 162 162 278 278 278 F F F 4 080-099 167 167 167 285 285 285 T F T <--- 5 100-119 158 158 158 270 270 270 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 166 166 166 282 282 282 T F T <--- 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 168 162 168 286 278 286 F F F <--- 12 240-259 164 164 164 280 280 280 F F F 13 260-279 160 160 160 273 273 273 T F T <--- 14 280-299 166 166 166 284 284 284 F F F 15 300-319 167 167 167 285 285 285 T F T <--- 16 320-339 164 164 164 280 280 280 T F T <--- 17 340-359 160 160 160 273 273 273 F F F 18 360-379 168 160 168 286 273 286 F F F <--- 19 380-399 168 167 168 286 285 286 F F F <--- 20 400-419 167 167 167 285 285 285 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 166 166 166 284 284 284 F F F 23 460-479 166 166 166 284 284 284 F F F 24 480-499 164 164 164 280 280 280 F F F 25 500-519 166 166 166 284 284 284 F F F 26 520-539 168 166 168 286 284 286 F F F <--- 27 540-559 168 163 168 286 279 286 F F F <--- 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 168, Max flush: 286 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 284 284 284 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 168 168 168 286 286 286 F F F 3 060-079 162 162 162 278 278 278 F F F 4 080-099 169 169 169 287 287 287 F F F 5 100-119 158 158 158 270 270 270 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 169 169 169 287 287 287 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 164 164 164 280 280 280 F F F 13 260-279 169 169 169 287 287 287 F F F 14 280-299 166 166 166 284 284 284 F F F 15 300-319 169 169 169 287 287 287 F F F 16 320-339 169 169 169 287 287 287 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 168 168 168 286 286 286 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 167 167 167 285 285 285 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 166 166 166 284 284 284 F F F 23 460-479 166 166 166 284 284 284 F F F 24 480-499 164 164 164 280 280 280 F F F 25 500-519 166 166 166 284 284 284 F F F 26 520-539 168 168 168 286 286 286 F F F 27 540-559 168 168 168 286 286 286 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 169, Max flush: 287 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.815186905Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823167043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823454226Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823478723Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823487102Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823498631Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Crucible ca2fc976-c67d-4365-b972-caf16e38c2cd has session id: 7970b2c8-b364-475a-8462-f1486b1bfecf","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823864074Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Upstairs opts: Upstairs UUID: ca2fc976-c67d-4365-b972-caf16e38c2cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823932346Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Crucible stats registered with UUID: ca2fc976-c67d-4365-b972-caf16e38c2cd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.823942784Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:17.823951793Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824104464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.82413559Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824146358Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824255025Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824270673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"ca2fc976-c67d-4365-b972-caf16e38c2cd active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.82429413Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824303428Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"0","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824313047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"1","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824322576Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"2","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824331975Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824341894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824351862Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:44460)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824414374Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58733)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824425053Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:52992)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.824435132Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.82515873Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"1","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825180267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"0","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825255918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"2","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825606853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[0]R flush_numbers[0..12]: [284, 286, 286, 278, 287, 270, 283, 287, 278, 282, 278, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825642589Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[0]R generation[0..12]: [166, 168, 168, 162, 169, 158, 166, 169, 162, 166, 162, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825653667Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825664466Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[1]R flush_numbers[0..12]: [284, 286, 286, 278, 287, 270, 283, 287, 278, 282, 278, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825674855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[1]R generation[0..12]: [166, 168, 168, 162, 169, 158, 166, 169, 162, 166, 162, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825685153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825695922Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[2]R flush_numbers[0..12]: [284, 286, 286, 278, 287, 270, 283, 287, 278, 282, 278, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825706381Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[2]R generation[0..12]: [166, 168, 168, 162, 169, 158, 166, 169, 162, 166, 162, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825716819Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825727498Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825736997Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825746715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825757794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825767133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"":"downstairs","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825776832Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.82578558Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"0","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825795059Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"1","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825804478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"client":"2","":"downstairs","session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825814147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825823206Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"ca2fc976-c67d-4365-b972-caf16e38c2cd is now active with session: 7970b2c8-b364-475a-8462-f1486b1bfecf","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825832894Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825841583Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"session_id":"7970b2c8-b364-475a-8462-f1486b1bfecf"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825850672Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Activated sub_volume ca2fc976-c67d-4365-b972-caf16e38c2cd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.825858591Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-10-01T01:05:17.82586679Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.826125837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"task":"crutest"} 01/30 Write block 339 len 4096 data: 10 02/30 Flush 03/30 Write block 299 len 24576 data: 10 11 13 13 16 16 04/30 Read block 183 len 16384 05/30 Read block 491 len 12288 06/30 Write block 244 len 28672 data: 9 13 12 15 16 17 13 07/30 Read block 111 len 32768 08/30 Flush 09/30 Write block 105 len 28672 data: 12 12 16 17 16 15 14 10/30 Read block 311 len 28672 11/30 Write block 458 len 8192 data: 15 18 12/30 Read block 138 len 32768 13/30 Write block 287 len 8192 data: 15 11 14/30 Read block 509 len 28672 15/30 Write block 63 len 12288 data: 12 11 11 16/30 Read block 404 len 28672 17/30 Read block 411 len 24576 18/30 Read block 184 len 4096 19/30 Write block 368 len 40960 data: 10 11 14 12 13 13 13 15 16 17 20/30 Read block 200 len 24576 21/30 Flush 22/30 Write block 50 len 28672 data: 17 18 14 15 17 17 17 23/30 Read block 249 len 20480 24/30 Flush 25/30 Write block 503 len 32768 data: 13 16 19 17 18 18 13 14 26/30 Write block 4 len 16384 data: 3 3 3 5 27/30 Write block 407 len 4096 data: 13 28/30 Write block 76 len 12288 data: 18 18 16 29/30 Write block 546 len 32768 data: 11 12 11 12 13 11 10 10 30/30 Read block 53 len 24576 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 30 32 Done 30 28 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:17.835057295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:17.83525046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1900,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 284 284 284 T T F <--- 1 020-039 168 168 168 286 286 286 F F F 2 040-059 170 170 168 291 291 286 F F F <--- 3 060-079 170 170 162 290 290 278 T T F <--- 4 080-099 169 169 169 287 287 287 F F F 5 100-119 170 170 158 290 290 270 F F F <--- 6 120-139 166 166 166 283 283 283 F F F 7 140-159 169 169 169 287 287 287 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 170 170 164 289 289 280 F F F <--- 13 260-279 169 169 169 287 287 287 F F F 14 280-299 170 170 166 290 290 284 F F F <--- 15 300-319 170 170 169 289 289 287 F F F <--- 16 320-339 170 170 169 288 288 287 F F F <--- 17 340-359 160 160 160 273 273 273 F F F 18 360-379 170 170 168 290 290 286 F F F <--- 19 380-399 168 168 168 286 286 286 F F F 20 400-419 167 167 167 285 285 285 T T F <--- 21 420-439 164 164 164 280 280 280 F F F 22 440-459 170 170 166 290 290 284 F F F <--- 23 460-479 166 166 166 284 284 284 F F F 24 480-499 164 164 164 280 280 280 F F F 25 500-519 166 166 166 284 284 284 T T F <--- 26 520-539 168 168 168 286 286 286 F F F 27 540-559 168 168 168 286 286 286 T T F <--- 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 170, Max flush: 291 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 292 292 292 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 170 170 170 291 291 291 F F F 3 060-079 171 171 171 292 292 292 F F F 4 080-099 169 169 169 287 287 287 F F F 5 100-119 170 170 170 290 290 290 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 169 169 169 287 287 287 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 162 162 162 278 278 278 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 169 169 169 287 287 287 F F F 14 280-299 170 170 170 290 290 290 F F F 15 300-319 170 170 170 289 289 289 F F F 16 320-339 170 170 170 288 288 288 F F F 17 340-359 160 160 160 273 273 273 F F F 18 360-379 170 170 170 290 290 290 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 170 170 170 290 290 290 F F F 23 460-479 166 166 166 284 284 284 F F F 24 480-499 164 164 164 280 280 280 F F F 25 500-519 171 171 171 292 292 292 F F F 26 520-539 168 168 168 286 286 286 F F F 27 540-559 171 171 171 292 292 292 F F F 28 560-579 162 162 162 278 278 278 F F F 29 580-599 161 161 161 275 275 275 F F F Max gen: 171, Max flush: 292 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.095135695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103101495Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103383029Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103407616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103416215Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103427633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Crucible a1ba994a-186c-4ec0-ac66-aafdbb500880 has session id: a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103808555Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Upstairs opts: Upstairs UUID: a1ba994a-186c-4ec0-ac66-aafdbb500880, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103860258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Crucible stats registered with UUID: a1ba994a-186c-4ec0-ac66-aafdbb500880","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.103873837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:32.103882865Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104013809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104042355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104053044Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104178108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104193166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"a1ba994a-186c-4ec0-ac66-aafdbb500880 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104225612Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104234851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"0","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104246299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"1","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104255918Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"2","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104265267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104275046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104284884Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38423)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104349636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62929)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104374773Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33879)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.104385982Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105061146Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"0","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105102461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"1","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105113739Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"2","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105433119Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[0]R flush_numbers[0..12]: [292, 286, 291, 292, 287, 290, 283, 287, 278, 282, 278, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105452106Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[0]R generation[0..12]: [171, 168, 170, 171, 169, 170, 166, 169, 162, 166, 162, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105462785Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105473724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[1]R flush_numbers[0..12]: [292, 286, 291, 292, 287, 290, 283, 287, 278, 282, 278, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105487352Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[1]R generation[0..12]: [171, 168, 170, 171, 169, 170, 166, 169, 162, 166, 162, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105497971Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105508699Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[2]R flush_numbers[0..12]: [292, 286, 291, 292, 287, 290, 283, 287, 278, 282, 278, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105519128Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[2]R generation[0..12]: [171, 168, 170, 171, 169, 170, 166, 169, 162, 166, 162, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105529467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105540185Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105549634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105559283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105568542Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105577761Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"":"downstairs","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105587449Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105596108Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"0","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105605647Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"1","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105615266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"client":"2","":"downstairs","session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105626564Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105635503Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"a1ba994a-186c-4ec0-ac66-aafdbb500880 is now active with session: a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105645362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105653991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"session_id":"a7a7cbd3-4fff-45e0-8dbe-39e2204dc16c"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.10566296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Activated sub_volume a1ba994a-186c-4ec0-ac66-aafdbb500880","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105671199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105679288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.105981559Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"task":"crutest"} 01/30 Write block 458 len 40960 data: 16 19 17 16 18 22 15 16 14 11 02/30 Write block 582 len 12288 data: 12 12 10 03/30 Read block 539 len 36864 04/30 Write block 456 len 16384 data: 15 16 17 20 05/30 Write block 539 len 36864 data: 20 21 19 15 11 13 12 12 13 06/30 Write block 578 len 28672 data: 11 10 11 13 13 13 11 07/30 Write block 542 len 28672 data: 16 12 14 13 13 14 12 08/30 Read block 570 len 36864 09/30 Write block 336 len 28672 data: 12 14 12 11 10 12 13 10/30 Read block 268 len 32768 11/30 Read block 555 len 12288 12/30 Read block 264 len 40960 13/30 Write block 492 len 32768 data: 11 13 9 8 8 6 8 9 14/30 Flush 15/30 Write block 452 len 28672 data: 11 15 17 14 16 17 18 16/30 Read block 39 len 16384 17/30 Write block 477 len 4096 data: 10 18/30 Read block 4 len 32768 19/30 Write block 205 len 12288 data: 17 18 20 20/30 Write block 69 len 16384 data: 9 9 14 15 21/30 Read block 248 len 32768 22/30 Write block 154 len 16384 data: 19 18 18 15 23/30 Write block 572 len 36864 data: 5 6 9 12 13 12 12 11 12 24/30 Read block 151 len 24576 25/30 Read block 101 len 4096 26/30 Read block 470 len 32768 27/30 Read block 317 len 16384 28/30 Write block 565 len 40960 data: 8 9 9 10 12 8 7 6 7 10 29/30 Write block 495 len 12288 data: 9 9 7 30/30 Read block 169 len 40960 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 2 27 Done 30 5 28 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:32.115178513Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:32.11535797Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1909,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 292 292 292 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 170 170 170 291 291 291 F F F 3 060-079 171 171 171 292 292 292 T F T <--- 4 080-099 169 169 169 287 287 287 F F F 5 100-119 170 170 170 290 290 290 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 169 169 169 287 287 287 T F T <--- 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 162 162 162 278 278 278 T F T <--- 11 220-239 168 168 168 286 286 286 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 169 169 169 287 287 287 F F F 14 280-299 170 170 170 290 290 290 F F F 15 300-319 170 170 170 289 289 289 F F F 16 320-339 172 170 172 293 288 293 F F F <--- 17 340-359 172 160 172 293 273 293 F F F <--- 18 360-379 170 170 170 290 290 290 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 172 170 172 293 290 293 T T T <--- 23 460-479 172 166 172 293 284 293 T T T <--- 24 480-499 172 164 172 293 280 293 T F T <--- 25 500-519 171 171 171 292 292 292 F F F 26 520-539 172 168 172 293 286 293 F T F <--- 27 540-559 172 171 172 293 292 293 F T F <--- 28 560-579 172 162 172 293 278 293 T F T <--- 29 580-599 172 161 172 293 275 293 T T T <--- Max gen: 172, Max flush: 293 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 292 292 292 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 170 170 170 291 291 291 F F F 3 060-079 173 173 173 294 294 294 F F F 4 080-099 169 169 169 287 287 287 F F F 5 100-119 170 170 170 290 290 290 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 173 173 173 294 294 294 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 173 173 173 294 294 294 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 169 169 169 287 287 287 F F F 14 280-299 170 170 170 290 290 290 F F F 15 300-319 170 170 170 289 289 289 F F F 16 320-339 172 172 172 293 293 293 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 170 170 170 290 290 290 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 173 173 173 294 294 294 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 171 171 171 292 292 292 F F F 26 520-539 172 172 172 293 293 293 F F F 27 540-559 172 172 172 293 293 293 F F F 28 560-579 173 173 173 294 294 294 F F F 29 580-599 173 173 173 294 294 294 F F F Max gen: 173, Max flush: 294 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.373536641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.381573612Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.381860695Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.381886322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.381894981Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.38190656Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Crucible a51d53be-e628-4b7d-968c-960af2f2af3d has session id: 52a78f9c-5169-4fca-8ab0-6c8f47dd0646","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382303209Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Upstairs opts: Upstairs UUID: a51d53be-e628-4b7d-968c-960af2f2af3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382356063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Crucible stats registered with UUID: a51d53be-e628-4b7d-968c-960af2f2af3d","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382369651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:46.38237875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382475777Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382507593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382518622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382631578Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382647496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"a51d53be-e628-4b7d-968c-960af2f2af3d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382679911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.38268931Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"0","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382699179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"1","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382708718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"2","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382718337Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382750972Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.38277109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"ds_connection connected from Ok(127.0.0.1:34629)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382782339Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"ds_connection connected from Ok(127.0.0.1:39116)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382888235Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"ds_connection connected from Ok(127.0.0.1:47450)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.382899954Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383534743Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"0","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383554991Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"1","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383572358Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"2","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383926283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[0]R flush_numbers[0..12]: [292, 286, 291, 294, 287, 290, 283, 294, 278, 282, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.38395196Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[0]R generation[0..12]: [171, 168, 170, 173, 169, 170, 166, 173, 162, 166, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383963229Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383974067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[1]R flush_numbers[0..12]: [292, 286, 291, 294, 287, 290, 283, 294, 278, 282, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383984616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[1]R generation[0..12]: [171, 168, 170, 173, 169, 170, 166, 173, 162, 166, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.383995135Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384006103Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[2]R flush_numbers[0..12]: [292, 286, 291, 294, 287, 290, 283, 294, 278, 282, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384016862Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[2]R generation[0..12]: [171, 168, 170, 173, 169, 170, 166, 173, 162, 166, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.38402962Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384040379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384049818Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384059447Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384068665Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384077824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"":"downstairs","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384087383Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384096122Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"0","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384105631Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"1","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384115179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"client":"2","":"downstairs","session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384125008Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384134067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"a51d53be-e628-4b7d-968c-960af2f2af3d is now active with session: 52a78f9c-5169-4fca-8ab0-6c8f47dd0646","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384143996Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384152565Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"session_id":"52a78f9c-5169-4fca-8ab0-6c8f47dd0646"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384161514Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Activated sub_volume a51d53be-e628-4b7d-968c-960af2f2af3d","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384169333Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384178801Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.384478633Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 102 len 12288 data: 13 13 12 04/30 Write block 586 len 20480 data: 8 8 9 9 12 05/30 Read block 390 len 36864 06/30 Read block 436 len 4096 07/30 Read block 346 len 8192 08/30 Read block 371 len 8192 09/30 Write block 285 len 20480 data: 15 17 16 12 11 10/30 Read block 436 len 40960 11/30 Read block 137 len 8192 12/30 Read block 424 len 4096 13/30 Read block 591 len 12288 14/30 Read block 357 len 12288 15/30 Write block 264 len 16384 data: 12 10 10 8 16/30 Read block 8 len 20480 17/30 Read block 394 len 32768 18/30 Write block 11 len 16384 data: 16 17 12 13 19/30 Read block 276 len 32768 20/30 Write block 311 len 24576 data: 15 13 11 13 15 14 21/30 Write block 46 len 12288 data: 14 14 13 22/30 Write block 267 len 8192 data: 9 9 23/30 Read block 161 len 24576 24/30 Flush 25/30 Write block 9 len 24576 data: 7 11 17 18 13 14 26/30 Read block 526 len 12288 27/30 Write block 338 len 4096 data: 13 28/30 Write block 362 len 28672 data: 13 14 14 15 15 13 11 29/30 Write block 556 len 28672 data: 7 8 8 7 5 5 5 30/30 Read block 275 len 36864 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 3 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 29 28 0 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:05:46.392869859Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:05:46.393054886Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1918,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 171 297 297 292 T T F <--- 1 020-039 168 168 168 286 286 286 F F F 2 040-059 174 174 170 297 297 291 F F F <--- 3 060-079 173 173 173 294 294 294 F F F 4 080-099 169 169 169 287 287 287 F F F 5 100-119 174 174 170 297 297 290 F F F <--- 6 120-139 166 166 166 283 283 283 F F F 7 140-159 173 173 173 294 294 294 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 173 173 173 294 294 294 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 174 174 169 297 297 287 F F F <--- 14 280-299 174 174 170 297 297 290 F F F <--- 15 300-319 174 174 170 297 297 289 F F F <--- 16 320-339 172 172 172 293 293 293 T T F <--- 17 340-359 172 172 172 293 293 293 F F F 18 360-379 170 170 170 290 290 290 T T F <--- 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 173 173 173 294 294 294 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 171 171 171 292 292 292 F F F 26 520-539 172 172 172 293 293 293 F F F 27 540-559 172 172 172 293 293 293 T T F <--- 28 560-579 173 173 173 294 294 294 T T F <--- 29 580-599 174 174 173 297 297 294 F F F <--- Max gen: 174, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 174 174 174 297 297 297 F F F 3 060-079 173 173 173 294 294 294 F F F 4 080-099 169 169 169 287 287 287 F F F 5 100-119 174 174 174 297 297 297 F F F 6 120-139 166 166 166 283 283 283 F F F 7 140-159 173 173 173 294 294 294 F F F 8 160-179 162 162 162 278 278 278 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 173 173 173 294 294 294 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 174 174 174 297 297 297 F F F 14 280-299 174 174 174 297 297 297 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 175 175 175 298 298 298 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 175 175 175 298 298 298 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 173 173 173 294 294 294 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 171 171 171 292 292 292 F F F 26 520-539 172 172 172 293 293 293 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 174 174 174 297 297 297 F F F Max gen: 175, Max flush: 298 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.651169985Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659144363Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659428617Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659454034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659462863Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659474371Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Crucible cc11d6d2-bb42-4ade-bc81-8a0cdd2f2c40 has session id: 43dbd8c4-2b98-4e88-a1e3-ef992e75fca5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659857003Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Upstairs opts: Upstairs UUID: cc11d6d2-bb42-4ade-bc81-8a0cdd2f2c40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659885009Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Crucible stats registered with UUID: cc11d6d2-bb42-4ade-bc81-8a0cdd2f2c40","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.659899178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:00.659908606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660042439Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660073605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660084504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660218437Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660235115Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"cc11d6d2-bb42-4ade-bc81-8a0cdd2f2c40 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660252463Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660261282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"0","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.66027084Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"1","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660280399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"2","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660290078Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660300047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660309815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53324)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660348091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33851)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660376637Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33746)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.660448968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.66106473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"2","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661085947Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"1","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661111644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"0","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661400577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[0]R flush_numbers[0..12]: [298, 286, 297, 294, 287, 297, 283, 294, 278, 282, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661433673Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[0]R generation[0..12]: [175, 168, 174, 173, 169, 174, 166, 173, 162, 166, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661444712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.66145556Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[1]R flush_numbers[0..12]: [298, 286, 297, 294, 287, 297, 283, 294, 278, 282, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661466049Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[1]R generation[0..12]: [175, 168, 174, 173, 169, 174, 166, 173, 162, 166, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661476567Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661487236Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[2]R flush_numbers[0..12]: [298, 286, 297, 294, 287, 297, 283, 294, 278, 282, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661497725Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[2]R generation[0..12]: [175, 168, 174, 173, 169, 174, 166, 173, 162, 166, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661508093Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661518712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661527951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.66153754Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661546749Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661556267Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"":"downstairs","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661567776Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661576525Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"0","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661586264Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"1","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661595822Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"client":"2","":"downstairs","session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661605481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.66161456Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"cc11d6d2-bb42-4ade-bc81-8a0cdd2f2c40 is now active with session: 43dbd8c4-2b98-4e88-a1e3-ef992e75fca5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661624399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661632978Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"session_id":"43dbd8c4-2b98-4e88-a1e3-ef992e75fca5"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661641957Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Activated sub_volume cc11d6d2-bb42-4ade-bc81-8a0cdd2f2c40","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661649856Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661658014Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.661973434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"task":"crutest"} 01/30 Write block 244 len 16384 data: 10 14 13 16 02/30 Flush 03/30 Flush 04/30 Write block 76 len 16384 data: 19 19 17 17 05/30 Read block 48 len 4096 06/30 Read block 350 len 24576 07/30 Read block 450 len 12288 08/30 Write block 154 len 32768 data: 20 19 19 16 14 11 11 12 09/30 Read block 205 len 24576 10/30 Write block 524 len 8192 data: 13 18 11/30 Flush 12/30 Write block 241 len 40960 data: 8 10 8 11 15 14 17 17 18 14 13/30 Read block 363 len 32768 14/30 Flush 15/30 Write block 129 len 40960 data: 20 19 22 24 19 19 16 16 14 12 16/30 Read block 526 len 32768 17/30 Write block 184 len 4096 data: 15 18/30 Read block 576 len 40960 19/30 Flush 20/30 Write block 505 len 16384 data: 20 18 19 19 21/30 Read block 505 len 24576 22/30 Read block 264 len 8192 23/30 Write block 248 len 8192 data: 18 19 24/30 Read block 96 len 8192 25/30 Read block 323 len 8192 26/30 Read block 162 len 12288 27/30 Write block 322 len 20480 data: 11 12 10 11 9 28/30 Read block 75 len 36864 29/30 Write block 261 len 24576 data: 10 13 10 13 11 11 30/30 Write block 54 len 4096 data: 18 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 24 2 28 Done 25 3 25 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1002)) Some(JobId(1018)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:00.671070941Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:00.671250958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1927,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 174 174 174 297 297 297 T F T <--- 3 060-079 176 173 176 301 294 301 F T F <--- 4 080-099 169 169 169 287 287 287 F F F 5 100-119 174 174 174 297 297 297 F F F 6 120-139 176 166 176 303 283 303 F F F <--- 7 140-159 176 173 176 301 294 301 F F F <--- 8 160-179 176 162 176 301 278 301 F F F <--- 9 180-199 176 166 176 303 282 303 F F F <--- 10 200-219 173 173 173 294 294 294 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 176 176 176 302 299 302 T F T <--- 13 260-279 174 174 174 297 297 297 T F T <--- 14 280-299 174 174 174 297 297 297 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 175 175 175 298 298 298 T F T <--- 17 340-359 172 172 172 293 293 293 F F F 18 360-379 175 175 175 298 298 298 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 173 173 173 294 294 294 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 171 171 171 292 292 292 T F T <--- 26 520-539 176 172 176 301 293 301 F F F <--- 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 174 174 174 297 297 297 F F F Max gen: 176, Max flush: 303 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 177 177 177 304 304 304 F F F 3 060-079 176 176 176 301 301 301 F F F 4 080-099 169 169 169 287 287 287 F F F 5 100-119 174 174 174 297 297 297 F F F 6 120-139 176 176 176 303 303 303 F F F 7 140-159 176 176 176 301 301 301 F F F 8 160-179 176 176 176 301 301 301 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 173 173 173 294 294 294 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 174 174 174 297 297 297 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 177 177 177 304 304 304 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 175 175 175 298 298 298 F F F 19 380-399 168 168 168 286 286 286 F F F 20 400-419 171 171 171 292 292 292 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 173 173 173 294 294 294 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 177 177 177 304 304 304 F F F 26 520-539 176 176 176 301 301 301 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 174 174 174 297 297 297 F F F Max gen: 177, Max flush: 304 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.93193527Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.939891021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940166507Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940193313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940201872Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.94021652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Crucible b87e5a72-9af8-47c4-8505-f265667500b0 has session id: 83672c2c-b501-4bd5-a0f8-0e3dfeb4e990","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940545259Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Upstairs opts: Upstairs UUID: b87e5a72-9af8-47c4-8505-f265667500b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940593012Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Crucible stats registered with UUID: b87e5a72-9af8-47c4-8505-f265667500b0","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940606601Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:14.94061563Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940756462Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.94077246Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940782478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940921391Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940937919Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"b87e5a72-9af8-47c4-8505-f265667500b0 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940953297Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940961956Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"0","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940971464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"1","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940981013Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"2","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.940990562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941012579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941022588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"ds_connection connected from Ok(127.0.0.1:51194)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.94108598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"ds_connection connected from Ok(127.0.0.1:42294)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941111367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"ds_connection connected from Ok(127.0.0.1:58526)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941123195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941616313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"0","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941653648Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"1","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.941751476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"2","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942012712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[0]R flush_numbers[0..12]: [298, 286, 304, 301, 287, 297, 303, 301, 301, 303, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942023851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[0]R generation[0..12]: [175, 168, 177, 176, 169, 174, 176, 176, 176, 176, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.94203468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942045788Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[1]R flush_numbers[0..12]: [298, 286, 304, 301, 287, 297, 303, 301, 301, 303, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942056317Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[1]R generation[0..12]: [175, 168, 177, 176, 169, 174, 176, 176, 176, 176, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942066776Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942079864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[2]R flush_numbers[0..12]: [298, 286, 304, 301, 287, 297, 303, 301, 301, 303, 294, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942090333Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[2]R generation[0..12]: [175, 168, 177, 176, 169, 174, 176, 176, 176, 176, 173, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942100631Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.94211136Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942120719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942130407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942139616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942148945Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"":"downstairs","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942158754Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942167373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"0","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942176762Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"1","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.94218628Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"client":"2","":"downstairs","session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942196099Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942205088Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"b87e5a72-9af8-47c4-8505-f265667500b0 is now active with session: 83672c2c-b501-4bd5-a0f8-0e3dfeb4e990","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942215047Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942223756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"session_id":"83672c2c-b501-4bd5-a0f8-0e3dfeb4e990"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942233874Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Activated sub_volume b87e5a72-9af8-47c4-8505-f265667500b0","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942241803Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942249812Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.942548544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"task":"crutest"} 01/30 Write block 590 len 32768 data: 13 14 12 11 11 12 10 8 02/30 Read block 337 len 28672 03/30 Read block 164 len 16384 04/30 Write block 205 len 40960 data: 18 19 21 22 22 20 18 20 18 18 05/30 Read block 187 len 24576 06/30 Write block 209 len 32768 data: 23 21 19 21 19 19 16 15 07/30 Flush 08/30 Flush 09/30 Flush 10/30 Write block 455 len 32768 data: 15 17 18 19 21 18 17 19 11/30 Write block 152 len 8192 data: 18 18 12/30 Write block 145 len 20480 data: 15 18 17 20 22 13/30 Read block 308 len 20480 14/30 Read block 463 len 24576 15/30 Read block 487 len 16384 16/30 Read block 512 len 12288 17/30 Read block 502 len 32768 18/30 Read block 81 len 40960 19/30 Read block 344 len 40960 20/30 Write block 395 len 32768 data: 11 9 10 8 9 11 12 16 21/30 Read block 534 len 32768 22/30 Write block 335 len 8192 data: 10 13 23/30 Read block 170 len 24576 24/30 Write block 90 len 24576 data: 14 12 8 10 9 11 25/30 Read block 466 len 8192 26/30 Write block 456 len 16384 data: 18 19 20 22 27/30 Read block 81 len 12288 28/30 Read block 274 len 4096 29/30 Read block 158 len 16384 30/30 Read block 575 len 16384 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Read 7 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:14.95149352Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:14.951675427Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1936,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 177 177 177 304 304 304 F F F 3 060-079 176 176 176 301 301 301 F F F 4 080-099 169 169 169 287 287 287 T T F <--- 5 100-119 174 174 174 297 297 297 F F F 6 120-139 176 176 176 303 303 303 F F F 7 140-159 176 176 176 301 301 301 T T F <--- 8 160-179 176 176 176 301 301 301 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 178 178 173 305 305 294 F F F <--- 11 220-239 168 168 168 286 286 286 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 174 174 174 297 297 297 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 177 177 177 304 304 304 T T F <--- 17 340-359 172 172 172 293 293 293 F F F 18 360-379 175 175 175 298 298 298 F F F 19 380-399 168 168 168 286 286 286 T T F <--- 20 400-419 171 171 171 292 292 292 T T F <--- 21 420-439 164 164 164 280 280 280 F F F 22 440-459 173 173 173 294 294 294 T T F <--- 23 460-479 173 173 173 294 294 294 T T F <--- 24 480-499 173 173 173 294 294 294 F F F 25 500-519 177 177 177 304 304 304 F F F 26 520-539 176 176 176 301 301 301 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 178 178 174 305 305 297 F F F <--- Max gen: 178, Max flush: 305 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 168 168 168 286 286 286 F F F 2 040-059 177 177 177 304 304 304 F F F 3 060-079 176 176 176 301 301 301 F F F 4 080-099 179 179 179 306 306 306 F F F 5 100-119 174 174 174 297 297 297 F F F 6 120-139 176 176 176 303 303 303 F F F 7 140-159 179 179 179 306 306 306 F F F 8 160-179 176 176 176 301 301 301 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 178 178 178 305 305 305 F F F 11 220-239 168 168 168 286 286 286 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 174 174 174 297 297 297 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 175 175 175 298 298 298 F F F 19 380-399 179 179 179 306 306 306 F F F 20 400-419 179 179 179 306 306 306 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 179 179 179 306 306 306 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 177 177 177 304 304 304 F F F 26 520-539 176 176 176 301 301 301 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 178 178 178 305 305 305 F F F Max gen: 179, Max flush: 306 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.204743496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:29.211715832Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.211768775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.218574852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.218832719Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.218857626Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.218866095Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.218877493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Crucible 93a36119-73a6-42b0-831d-a76a5df11946 has session id: 352c1ca4-5a26-42fd-a11f-f0e361ad138c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219215011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Upstairs opts: Upstairs UUID: 93a36119-73a6-42b0-831d-a76a5df11946, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219244887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Crucible stats registered with UUID: 93a36119-73a6-42b0-831d-a76a5df11946","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219258915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:29.219270254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219386799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219403787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219413845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219576735Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219600902Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"93a36119-73a6-42b0-831d-a76a5df11946 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.21961394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219623109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"0","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219632638Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"1","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219642166Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"2","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219651585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219661704Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219671683Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42266)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219757002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59215)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.219782169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41687)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.21985261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220420738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"1","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220432956Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"0","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220444195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"2","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220770313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[0]R flush_numbers[0..12]: [298, 286, 304, 301, 306, 297, 303, 306, 301, 303, 305, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220786481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[0]R generation[0..12]: [175, 168, 177, 176, 179, 174, 176, 179, 176, 176, 178, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.22079752Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220808539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[1]R flush_numbers[0..12]: [298, 286, 304, 301, 306, 297, 303, 306, 301, 303, 305, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220819167Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[1]R generation[0..12]: [175, 168, 177, 176, 179, 174, 176, 179, 176, 176, 178, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220829696Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220840385Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[2]R flush_numbers[0..12]: [298, 286, 304, 301, 306, 297, 303, 306, 301, 303, 305, 286]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220850903Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[2]R generation[0..12]: [175, 168, 177, 176, 179, 174, 176, 179, 176, 176, 178, 168]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220861342Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.22087222Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220884509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220894168Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220903526Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220912925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"":"downstairs","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220922554Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220931193Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"0","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220940792Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"1","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220950381Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"client":"2","":"downstairs","session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220960089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220969028Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"93a36119-73a6-42b0-831d-a76a5df11946 is now active with session: 352c1ca4-5a26-42fd-a11f-f0e361ad138c","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220978857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220987516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"session_id":"352c1ca4-5a26-42fd-a11f-f0e361ad138c"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.220996505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Activated sub_volume 93a36119-73a6-42b0-831d-a76a5df11946","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.221004394Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.221012493Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.221339091Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} 01/30 Write block 121 len 24576 data: 11 17 18 19 20 21 02/30 Read block 499 len 12288 03/30 Read block 564 len 12288 04/30 Read block 29 len 36864 05/30 Read block 518 len 12288 06/30 Read block 540 len 32768 07/30 Write block 171 len 12288 data: 18 21 19 08/30 Write block 102 len 16384 data: 14 14 13 13 09/30 Read block 131 len 40960 10/30 Write block 384 len 16384 data: 13 12 11 14 11/30 Flush 12/30 Write block 26 len 24576 data: 6 10 11 12 13 12 13/30 Write block 212 len 40960 data: 22 20 20 17 16 15 18 19 21 22 14/30 Read block 418 len 28672 15/30 Read block 307 len 12288 16/30 Write block 343 len 8192 data: 14 14 17/30 Write block 286 len 4096 data: 18 18/30 Flush 19/30 Write block 77 len 36864 data: 20 18 18 15 11 10 9 14 15 20/30 Read block 358 len 4096 21/30 Write block 402 len 8192 data: 17 15 22/30 Write block 156 len 4096 data: 20 23/30 Write block 282 len 20480 data: 16 18 14 16 19 24/30 Write block 373 len 28672 data: 14 14 16 17 18 15 15 25/30 Read block 135 len 36864 26/30 Flush 27/30 Write block 138 len 16384 data: 13 14 10 15 28/30 Write block 473 len 4096 data: 8 29/30 Write block 95 len 8192 data: 12 14 30/30 Read block 228 len 12288 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 5 30 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:29.2298654Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:29.230064525Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1945,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 168 180 180 286 308 308 F F F <--- 2 040-059 177 177 177 304 304 304 F F F 3 060-079 176 180 180 301 309 309 F F F <--- 4 080-099 179 180 180 306 309 309 F T T <--- 5 100-119 174 180 180 297 307 307 F F F <--- 6 120-139 176 180 180 303 307 307 T T T <--- 7 140-159 179 180 180 306 309 309 F T T <--- 8 160-179 176 180 180 301 307 307 F F F <--- 9 180-199 176 176 176 303 303 303 F F F 10 200-219 178 180 180 305 308 308 F F F <--- 11 220-239 168 180 180 286 308 308 F F F <--- 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 174 180 180 297 309 309 F F F <--- 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 172 180 180 293 308 308 F F F <--- 18 360-379 175 180 180 298 309 309 F F F <--- 19 380-399 179 180 180 306 307 307 F F F <--- 20 400-419 179 180 180 306 309 309 F F F <--- 21 420-439 164 164 164 280 280 280 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 179 179 179 306 306 306 F T T <--- 24 480-499 173 173 173 294 294 294 F F F 25 500-519 177 177 177 304 304 304 F F F 26 520-539 176 176 176 301 301 301 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 178 178 178 305 305 305 F F F Max gen: 180, Max flush: 309 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 180 180 180 308 308 308 F F F 2 040-059 177 177 177 304 304 304 F F F 3 060-079 180 180 180 309 309 309 F F F 4 080-099 181 181 181 310 310 310 F F F 5 100-119 180 180 180 307 307 307 F F F 6 120-139 181 181 181 310 310 310 F F F 7 140-159 181 181 181 310 310 310 F F F 8 160-179 180 180 180 307 307 307 F F F 9 180-199 176 176 176 303 303 303 F F F 10 200-219 180 180 180 308 308 308 F F F 11 220-239 180 180 180 308 308 308 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 180 180 180 309 309 309 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 180 180 180 308 308 308 F F F 18 360-379 180 180 180 309 309 309 F F F 19 380-399 180 180 180 307 307 307 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 164 164 164 280 280 280 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 177 177 177 304 304 304 F F F 26 520-539 176 176 176 301 301 301 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 175 175 175 298 298 298 F F F 29 580-599 178 178 178 305 305 305 F F F Max gen: 181, Max flush: 310 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.492606232Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.500622176Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.500895741Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.500922718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.500931386Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.500942835Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Crucible 8e146a2c-8c41-4aa1-b7ca-7b6fdc12bf48 has session id: 1bd7970a-2d6c-4d00-a40c-134688c95217","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501210511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Upstairs opts: Upstairs UUID: 8e146a2c-8c41-4aa1-b7ca-7b6fdc12bf48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501227109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Crucible stats registered with UUID: 8e146a2c-8c41-4aa1-b7ca-7b6fdc12bf48","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501239577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:43.501248566Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501407076Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"0","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501422604Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"1","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501432783Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"2","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501594622Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.50161355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"8e146a2c-8c41-4aa1-b7ca-7b6fdc12bf48 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501673582Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501682861Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"0","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.50169264Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"1","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501702309Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"2","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501711877Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"0","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501721746Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"2","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501731725Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"1","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"ds_connection connected from Ok(127.0.0.1:47610)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501799366Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"2","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"ds_connection connected from Ok(127.0.0.1:41412)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501824263Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"1","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"ds_connection connected from Ok(127.0.0.1:57826)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.501839851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"io task","client":"0","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502469261Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"2","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502505067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"1","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502605424Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"0","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502838295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[0]R flush_numbers[0..12]: [298, 308, 304, 309, 310, 307, 310, 310, 307, 303, 308, 308]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502865071Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[0]R generation[0..12]: [175, 180, 177, 180, 181, 180, 181, 181, 180, 176, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502879769Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502890868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[1]R flush_numbers[0..12]: [298, 308, 304, 309, 310, 307, 310, 310, 307, 303, 308, 308]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502901377Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[1]R generation[0..12]: [175, 180, 177, 180, 181, 180, 181, 181, 180, 176, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502911845Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502922574Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[2]R flush_numbers[0..12]: [298, 308, 304, 309, 310, 307, 310, 310, 307, 303, 308, 308]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502933043Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[2]R generation[0..12]: [175, 180, 177, 180, 181, 180, 181, 181, 180, 176, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502943511Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.50295422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502963889Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502973697Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502983046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.502992415Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"":"downstairs","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503014062Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503022811Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"0","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.5030325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"1","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503043808Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"client":"2","":"downstairs","session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503053627Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503062686Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"8e146a2c-8c41-4aa1-b7ca-7b6fdc12bf48 is now active with session: 1bd7970a-2d6c-4d00-a40c-134688c95217","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503072595Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503081234Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"session_id":"1bd7970a-2d6c-4d00-a40c-134688c95217"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503090243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Activated sub_volume 8e146a2c-8c41-4aa1-b7ca-7b6fdc12bf48","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503098102Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503106021Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.503404843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"task":"crutest"} 01/30 Write block 429 len 8192 data: 15 12 02/30 Write block 93 len 40960 data: 11 10 13 15 14 14 14 13 14 15 03/30 Write block 176 len 32768 data: 17 16 11 12 14 12 12 17 04/30 Read block 330 len 8192 05/30 Write block 504 len 12288 data: 17 21 19 06/30 Read block 309 len 36864 07/30 Write block 530 len 12288 data: 12 13 16 08/30 Read block 401 len 40960 09/30 Write block 561 len 12288 data: 6 6 7 10/30 Read block 379 len 24576 11/30 Write block 426 len 40960 data: 16 16 14 16 13 14 14 12 12 11 12/30 Write block 576 len 32768 data: 14 13 13 12 13 14 14 14 13/30 Write block 63 len 16384 data: 13 12 12 10 14/30 Flush 15/30 Flush 16/30 Write block 81 len 12288 data: 12 11 10 17/30 Read block 35 len 28672 18/30 Write block 359 len 20480 data: 12 12 14 14 15 19/30 Write block 578 len 32768 data: 14 13 14 15 15 15 12 9 20/30 Read block 43 len 36864 21/30 Write block 115 len 24576 data: 11 10 8 11 10 11 22/30 Write block 565 len 24576 data: 9 10 10 11 13 9 23/30 Read block 360 len 16384 24/30 Read block 227 len 4096 25/30 Read block 395 len 28672 26/30 Read block 339 len 4096 27/30 Read block 493 len 40960 28/30 Read block 422 len 8192 29/30 Write block 125 len 12288 data: 21 22 20 30/30 Read block 559 len 28672 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:43.511583275Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:43.511750544Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1954,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 180 180 180 308 308 308 F F F 2 040-059 177 177 177 304 304 304 F F F 3 060-079 182 180 182 311 309 311 F F F <--- 4 080-099 182 181 182 311 310 311 T F T <--- 5 100-119 182 180 182 311 307 311 T F T <--- 6 120-139 181 181 181 310 310 310 T F T <--- 7 140-159 181 181 181 310 310 310 F F F 8 160-179 182 180 182 311 307 311 F F F <--- 9 180-199 182 176 182 311 303 311 F F F <--- 10 200-219 180 180 180 308 308 308 F F F 11 220-239 180 180 180 308 308 308 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 180 180 180 309 309 309 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 180 180 180 308 308 308 T F T <--- 18 360-379 180 180 180 309 309 309 T F T <--- 19 380-399 180 180 180 307 307 307 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 164 182 311 280 311 F F F <--- 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 182 177 182 311 304 311 F F F <--- 26 520-539 182 176 182 311 301 311 F F F <--- 27 540-559 175 175 175 298 298 298 F F F 28 560-579 182 175 182 311 298 311 T F T <--- 29 580-599 182 178 182 311 305 311 T F T <--- Max gen: 182, Max flush: 311 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 180 180 180 308 308 308 F F F 2 040-059 177 177 177 304 304 304 F F F 3 060-079 182 182 182 311 311 311 F F F 4 080-099 183 183 183 312 312 312 F F F 5 100-119 183 183 183 312 312 312 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 181 181 181 310 310 310 F F F 8 160-179 182 182 182 311 311 311 F F F 9 180-199 182 182 182 311 311 311 F F F 10 200-219 180 180 180 308 308 308 F F F 11 220-239 180 180 180 308 308 308 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 F F F 14 280-299 180 180 180 309 309 309 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 183 183 183 312 312 312 F F F 18 360-379 183 183 183 312 312 312 F F F 19 380-399 180 180 180 307 307 307 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 173 173 173 294 294 294 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 182 182 182 311 311 311 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 183, Max flush: 312 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.77564601Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.783577484Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.783855509Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.783881366Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.783889925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.783901313Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Crucible 84991f10-c6b3-48bc-9850-ccff982b184a has session id: 4e3aeaee-0988-46c2-b3e8-cd85db5aabf4","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784217543Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Upstairs opts: Upstairs UUID: 84991f10-c6b3-48bc-9850-ccff982b184a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784246789Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Crucible stats registered with UUID: 84991f10-c6b3-48bc-9850-ccff982b184a","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784260068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:57.784269416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784391111Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784410809Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784420807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784569488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784582147Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"84991f10-c6b3-48bc-9850-ccff982b184a active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784619562Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784642799Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"0","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784653848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"1","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784663606Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"2","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784692933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.78471362Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784726768Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36908)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784737247Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40311)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784774202Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36651)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.784799179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785404423Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"0","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785448787Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"1","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785490182Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"2","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785789904Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[0]R flush_numbers[0..12]: [298, 308, 304, 311, 312, 312, 312, 310, 311, 311, 308, 308]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.78581843Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[0]R generation[0..12]: [175, 180, 177, 182, 183, 183, 183, 181, 182, 182, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785830139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785841087Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[1]R flush_numbers[0..12]: [298, 308, 304, 311, 312, 312, 312, 310, 311, 311, 308, 308]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785851756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[1]R generation[0..12]: [175, 180, 177, 182, 183, 183, 183, 181, 182, 182, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785862144Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785873143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[2]R flush_numbers[0..12]: [298, 308, 304, 311, 312, 312, 312, 310, 311, 311, 308, 308]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785886421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[2]R generation[0..12]: [175, 180, 177, 182, 183, 183, 183, 181, 182, 182, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.78589683Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785907639Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785917008Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785926636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785935915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785945284Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"":"downstairs","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785955053Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785963842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"0","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.7859737Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"1","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785983519Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"client":"2","":"downstairs","session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.785993178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.786013535Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"84991f10-c6b3-48bc-9850-ccff982b184a is now active with session: 4e3aeaee-0988-46c2-b3e8-cd85db5aabf4","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.786023524Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.786032173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"session_id":"4e3aeaee-0988-46c2-b3e8-cd85db5aabf4"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.786041282Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Activated sub_volume 84991f10-c6b3-48bc-9850-ccff982b184a","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.786050481Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.78605848Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.786342044Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"task":"crutest"} 01/30 Write block 387 len 4096 data: 15 02/30 Read block 138 len 40960 03/30 Read block 352 len 40960 04/30 Flush 05/30 Write block 147 len 20480 data: 18 21 23 19 15 06/30 Flush 07/30 Write block 343 len 16384 data: 15 15 17 19 08/30 Read block 478 len 20480 09/30 Read block 170 len 32768 10/30 Write block 41 len 24576 data: 9 11 10 13 11 15 11/30 Flush 12/30 Write block 47 len 40960 data: 15 14 18 18 19 15 16 19 18 18 13/30 Read block 493 len 8192 14/30 Read block 126 len 16384 15/30 Read block 232 len 4096 16/30 Flush 17/30 Write block 488 len 16384 data: 9 7 6 8 18/30 Write block 500 len 20480 data: 7 8 10 14 18 19/30 Read block 132 len 16384 20/30 Write block 61 len 32768 data: 17 13 14 13 13 11 11 10 21/30 Flush 22/30 Write block 56 len 16384 data: 19 18 17 16 23/30 Flush 24/30 Write block 502 len 16384 data: 11 15 19 22 25/30 Read block 105 len 8192 26/30 Read block 269 len 36864 27/30 Write block 229 len 36864 data: 8 10 12 10 10 10 12 13 12 28/30 Write block 94 len 8192 data: 11 14 29/30 Write block 276 len 28672 data: 16 17 15 15 17 17 17 30/30 Write block 20 len 24576 data: 6 5 5 4 5 6 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 10 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 29 4 37 Done 26 1 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:06:57.794667598Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:06:57.794829367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1963,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 180 180 180 308 308 308 T F T <--- 2 040-059 184 177 184 318 304 318 F F F <--- 3 060-079 184 182 184 317 311 317 F F F <--- 4 080-099 183 183 183 312 312 312 T F T <--- 5 100-119 183 183 183 312 312 312 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 184 181 184 314 310 314 F F F <--- 8 160-179 182 182 182 311 311 311 F F F 9 180-199 182 182 182 311 311 311 F F F 10 200-219 180 180 180 308 308 308 F F F 11 220-239 180 180 180 308 308 308 T F T <--- 12 240-259 177 177 177 304 304 304 F F F 13 260-279 177 177 177 304 304 304 T F T <--- 14 280-299 180 180 180 309 309 309 T F T <--- 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 184 183 184 315 312 315 F F F <--- 18 360-379 183 183 183 312 312 312 F F F 19 380-399 184 180 184 313 307 313 F T F <--- 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 184 173 184 317 294 317 F F F <--- 25 500-519 184 182 184 317 311 317 T F T <--- 26 520-539 182 182 182 311 311 311 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 184, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 298 298 298 F F F 1 020-039 185 185 185 319 319 319 F F F 2 040-059 184 184 184 318 318 318 F F F 3 060-079 184 184 184 317 317 317 F F F 4 080-099 185 185 185 319 319 319 F F F 5 100-119 183 183 183 312 312 312 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 184 184 184 314 314 314 F F F 8 160-179 182 182 182 311 311 311 F F F 9 180-199 182 182 182 311 311 311 F F F 10 200-219 180 180 180 308 308 308 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 185 185 185 319 319 319 F F F 14 280-299 185 185 185 319 319 319 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 184 184 184 315 315 315 F F F 18 360-379 183 183 183 312 312 312 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 184 184 184 317 317 317 F F F 25 500-519 185 185 185 319 319 319 F F F 26 520-539 182 182 182 311 311 311 F F F 27 540-559 175 175 175 298 298 298 F F F 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 185, Max flush: 319 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.065699698Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:12.072682272Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.072736506Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.079536153Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.079810958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.079835535Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.079844034Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.079855333Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Crucible 8b8c30c6-ccb3-4843-83d7-5edf1b07b4c3 has session id: 79c15205-c38e-4361-9e16-69ed32aad6e5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080244473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Upstairs opts: Upstairs UUID: 8b8c30c6-ccb3-4843-83d7-5edf1b07b4c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.08027334Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Crucible stats registered with UUID: 8b8c30c6-ccb3-4843-83d7-5edf1b07b4c3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080291227Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:12.080302616Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080436199Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080466815Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080477594Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080600968Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080616536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"8b8c30c6-ccb3-4843-83d7-5edf1b07b4c3 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080633504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080642283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"0","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080651952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"1","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.08066173Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"2","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080671579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080681588Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080706895Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61042)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080750399Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54809)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080761958Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:45061)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.080857846Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081465349Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"1","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081477897Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"2","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081538749Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"0","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081881296Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[0]R flush_numbers[0..12]: [298, 319, 318, 317, 319, 312, 312, 314, 311, 311, 308, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081907023Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[0]R generation[0..12]: [175, 185, 184, 184, 185, 183, 183, 184, 182, 182, 180, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081918181Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.08192909Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[1]R flush_numbers[0..12]: [298, 319, 318, 317, 319, 312, 312, 314, 311, 311, 308, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081939828Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[1]R generation[0..12]: [175, 185, 184, 184, 185, 183, 183, 184, 182, 182, 180, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081950307Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081961006Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[2]R flush_numbers[0..12]: [298, 319, 318, 317, 319, 312, 312, 314, 311, 311, 308, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081971444Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[2]R generation[0..12]: [175, 185, 184, 184, 185, 183, 183, 184, 182, 182, 180, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081981883Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.081992742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.08200204Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082013939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082023158Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082032467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"":"downstairs","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082041935Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082050674Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"0","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082060263Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"1","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082069762Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"client":"2","":"downstairs","session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082079541Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.08208853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"8b8c30c6-ccb3-4843-83d7-5edf1b07b4c3 is now active with session: 79c15205-c38e-4361-9e16-69ed32aad6e5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082098328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082111867Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"session_id":"79c15205-c38e-4361-9e16-69ed32aad6e5"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082120915Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Activated sub_volume 8b8c30c6-ccb3-4843-83d7-5edf1b07b4c3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082128774Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082136923Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.082455273Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} 01/30 Write block 159 len 20480 data: 12 12 13 11 14 02/30 Flush 03/30 Write block 556 len 12288 data: 8 9 9 04/30 Write block 199 len 12288 data: 11 12 9 05/30 Write block 4 len 36864 data: 4 4 4 6 7 8 12 18 19 06/30 Read block 36 len 28672 07/30 Read block 228 len 24576 08/30 Write block 500 len 12288 data: 8 9 12 09/30 Write block 490 len 24576 data: 7 9 12 14 10 10 10/30 Read block 101 len 36864 11/30 Write block 159 len 4096 data: 13 12/30 Flush 13/30 Write block 172 len 36864 data: 22 20 18 19 18 17 12 13 15 14/30 Read block 512 len 36864 15/30 Flush 16/30 Write block 512 len 4096 data: 15 17/30 Read block 544 len 24576 18/30 Read block 203 len 28672 19/30 Read block 250 len 8192 20/30 Read block 546 len 32768 21/30 Flush 22/30 Write block 152 len 16384 data: 19 19 21 20 23/30 Flush 24/30 Write block 52 len 8192 data: 16 17 25/30 Read block 60 len 20480 26/30 Write block 62 len 20480 data: 14 15 14 14 12 27/30 Read block 269 len 32768 28/30 Write block 359 len 20480 data: 13 13 15 15 16 29/30 Write block 199 len 4096 data: 12 30/30 Write block 31 len 28672 data: 13 8 8 8 7 7 8 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 3 3 33 Done 1 25 25 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:12.090945136Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:12.091159689Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1972,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 186 186 298 321 321 F F F <--- 1 020-039 185 185 185 319 319 319 F T T <--- 2 040-059 184 184 184 318 318 318 F T T <--- 3 060-079 184 184 184 317 317 317 F T T <--- 4 080-099 185 185 185 319 319 319 F F F 5 100-119 183 183 183 312 312 312 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 186 186 186 320 324 324 F F F <--- 8 160-179 186 186 186 320 322 322 F F F <--- 9 180-199 182 186 186 311 322 322 F T T <--- 10 200-219 180 186 186 308 321 321 F F F <--- 11 220-239 185 185 185 319 319 319 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 185 185 185 319 319 319 F F F 14 280-299 185 185 185 319 319 319 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 184 184 184 315 315 315 F T T <--- 18 360-379 183 183 183 312 312 312 F T T <--- 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 184 186 186 317 321 321 F F F <--- 25 500-519 185 186 186 319 323 323 F F F <--- 26 520-539 182 182 182 311 311 311 F F F 27 540-559 175 186 186 298 321 321 T F F <--- 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 186, Max flush: 324 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 321 321 321 F F F 1 020-039 187 187 187 325 325 325 F F F 2 040-059 187 187 187 325 325 325 F F F 3 060-079 187 187 187 325 325 325 F F F 4 080-099 185 185 185 319 319 319 F F F 5 100-119 183 183 183 312 312 312 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 186 186 186 324 324 324 F F F 8 160-179 186 186 186 322 322 322 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 177 177 177 304 304 304 F F F 13 260-279 185 185 185 319 319 319 F F F 14 280-299 185 185 185 319 319 319 F F F 15 300-319 174 174 174 297 297 297 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 187 187 187 325 325 325 F F F 18 360-379 187 187 187 325 325 325 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 179 179 179 306 306 306 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 186 186 186 321 321 321 F F F 25 500-519 186 186 186 323 323 323 F F F 26 520-539 182 182 182 311 311 311 F F F 27 540-559 186 186 186 321 321 321 F F F 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 187, Max flush: 325 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.372006205Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.379999001Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380299643Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.38032473Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380333429Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380345178Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"Crucible 07785bf4-e7c1-4db7-911a-a8a264504eb2 has session id: 7d39895a-1040-4edf-bc13-8d64348a24b4","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380706722Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Upstairs opts: Upstairs UUID: 07785bf4-e7c1-4db7-911a-a8a264504eb2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380737008Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Crucible stats registered with UUID: 07785bf4-e7c1-4db7-911a-a8a264504eb2","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380747986Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:26.380757195Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380889139Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"2","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380905467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"0","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.380915455Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"1","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381071445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381090953Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"07785bf4-e7c1-4db7-911a-a8a264504eb2 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381106851Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.38111766Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"0","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381127448Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"1","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381137127Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"2","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381146716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"2","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381156625Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"0","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381166623Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"1","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:48266)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381207098Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"2","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42609)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381218007Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"0","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44362)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381228286Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"io task","client":"1","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381863945Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"0","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381897201Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"1","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.381908329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"2","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382234938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[0]R flush_numbers[0..12]: [321, 325, 325, 325, 319, 312, 312, 324, 322, 325, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382262434Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[0]R generation[0..12]: [186, 187, 187, 187, 185, 183, 183, 186, 186, 187, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382273483Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382286911Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[1]R flush_numbers[0..12]: [321, 325, 325, 325, 319, 312, 312, 324, 322, 325, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.38229745Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[1]R generation[0..12]: [186, 187, 187, 187, 185, 183, 183, 186, 186, 187, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382307949Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382318837Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[2]R flush_numbers[0..12]: [321, 325, 325, 325, 319, 312, 312, 324, 322, 325, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382329266Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[2]R generation[0..12]: [186, 187, 187, 187, 185, 183, 183, 186, 186, 187, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382339435Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382350243Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382359552Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382369211Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.38237842Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382387839Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"":"downstairs","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382397467Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382406186Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"0","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382415795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"1","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382425314Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"client":"2","":"downstairs","session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382436942Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382445951Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"07785bf4-e7c1-4db7-911a-a8a264504eb2 is now active with session: 7d39895a-1040-4edf-bc13-8d64348a24b4","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.38245577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382464429Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"session_id":"7d39895a-1040-4edf-bc13-8d64348a24b4"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382473468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"Activated sub_volume 07785bf4-e7c1-4db7-911a-a8a264504eb2","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382481347Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382494445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.382744133Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"task":"crutest"} 01/30 Write block 69 len 36864 data: 10 10 15 16 13 16 19 20 21 02/30 Write block 182 len 24576 data: 13 18 16 15 16 16 03/30 Read block 233 len 12288 04/30 Read block 541 len 32768 05/30 Read block 19 len 36864 06/30 Write block 298 len 20480 data: 11 11 12 14 14 07/30 Read block 310 len 4096 08/30 Read block 230 len 8192 09/30 Write block 554 len 24576 data: 7 10 9 10 10 8 10/30 Flush 11/30 Write block 272 len 20480 data: 12 13 12 14 17 12/30 Read block 525 len 40960 13/30 Read block 306 len 4096 14/30 Write block 444 len 12288 data: 12 15 18 15/30 Read block 81 len 32768 16/30 Read block 81 len 24576 17/30 Write block 493 len 20480 data: 15 11 11 10 8 18/30 Write block 248 len 40960 data: 19 20 15 14 14 12 11 11 14 10 19/30 Read block 208 len 32768 20/30 Write block 87 len 24576 data: 15 21 18 15 13 9 21/30 Read block 112 len 36864 22/30 Read block 68 len 16384 23/30 Write block 143 len 24576 data: 14 16 16 19 19 22 24/30 Write block 545 len 36864 data: 14 14 15 13 13 14 12 11 11 25/30 Write block 17 len 16384 data: 12 10 7 7 26/30 Write block 0 len 12288 data: 3 3 4 27/30 Write block 115 len 12288 data: 12 11 9 28/30 Write block 255 len 28672 data: 12 15 11 13 12 11 11 29/30 Write block 53 len 12288 data: 18 20 19 30/30 Read block 522 len 40960 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:26.390876082Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:26.391075887Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1981,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 321 321 321 T T F <--- 1 020-039 187 187 187 325 325 325 T T F <--- 2 040-059 187 187 187 325 325 325 T T F <--- 3 060-079 188 188 187 326 326 325 F F T <--- 4 080-099 185 185 185 319 319 319 T T F <--- 5 100-119 183 183 183 312 312 312 T T F <--- 6 120-139 183 183 183 312 312 312 F F F 7 140-159 186 186 186 324 324 324 T T F <--- 8 160-179 186 186 186 322 322 322 F F F 9 180-199 188 188 187 326 326 325 F F F <--- 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 177 177 177 304 304 304 T T F <--- 13 260-279 185 185 185 319 319 319 T T F <--- 14 280-299 188 188 185 326 326 319 F F F <--- 15 300-319 188 188 174 326 326 297 F F F <--- 16 320-339 179 179 179 306 306 306 F F F 17 340-359 187 187 187 325 325 325 F F F 18 360-379 187 187 187 325 325 325 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 179 179 179 306 306 306 T T F <--- 23 460-479 181 181 181 310 310 310 F F F 24 480-499 186 186 186 321 321 321 T T F <--- 25 500-519 186 186 186 323 323 323 F F F 26 520-539 182 182 182 311 311 311 F F F 27 540-559 188 188 186 326 326 321 T T F <--- 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 188, Max flush: 326 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 327 327 327 F F F 1 020-039 189 189 189 327 327 327 F F F 2 040-059 189 189 189 327 327 327 F F F 3 060-079 188 188 188 326 326 326 F F F 4 080-099 189 189 189 327 327 327 F F F 5 100-119 189 189 189 327 327 327 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 189 189 189 327 327 327 F F F 8 160-179 186 186 186 322 322 322 F F F 9 180-199 188 188 188 326 326 326 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 189 189 189 327 327 327 F F F 13 260-279 189 189 189 327 327 327 F F F 14 280-299 188 188 188 326 326 326 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 179 179 179 306 306 306 F F F 17 340-359 187 187 187 325 325 325 F F F 18 360-379 187 187 187 325 325 325 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 189 189 189 327 327 327 F F F 23 460-479 181 181 181 310 310 310 F F F 24 480-499 189 189 189 327 327 327 F F F 25 500-519 186 186 186 323 323 323 F F F 26 520-539 182 182 182 311 311 311 F F F 27 540-559 189 189 189 327 327 327 F F F 28 560-579 183 183 183 312 312 312 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 189, Max flush: 327 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.653988487Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.661897984Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662184018Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662212224Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662238191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662249899Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Crucible f4e8bbd5-a2f8-4d88-a967-d3f2a0fd2c73 has session id: c5870fdf-ec01-4088-909d-8b6614027fdd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662572048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Upstairs opts: Upstairs UUID: f4e8bbd5-a2f8-4d88-a967-d3f2a0fd2c73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662602925Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Crucible stats registered with UUID: f4e8bbd5-a2f8-4d88-a967-d3f2a0fd2c73","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662613813Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:40.662622942Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662749236Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"0","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662768014Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"2","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662778172Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"1","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.662980167Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663005853Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"f4e8bbd5-a2f8-4d88-a967-d3f2a0fd2c73 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.66303115Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663040619Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"0","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663050488Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"1","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663060107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"2","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663069605Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"2","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663079464Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"0","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663094852Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"1","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65472)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663174212Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"2","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33414)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663198939Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"1","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33641)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663210388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"io task","client":"0","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663835878Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"1","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.663875443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"0","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.66390011Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"2","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664195833Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[0]R flush_numbers[0..12]: [327, 327, 327, 326, 327, 327, 312, 327, 322, 326, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.66421916Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[0]R generation[0..12]: [189, 189, 189, 188, 189, 189, 183, 189, 186, 188, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664231228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664242407Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[1]R flush_numbers[0..12]: [327, 327, 327, 326, 327, 327, 312, 327, 322, 326, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664253055Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[1]R generation[0..12]: [189, 189, 189, 188, 189, 189, 183, 189, 186, 188, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664263804Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664274783Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[2]R flush_numbers[0..12]: [327, 327, 327, 326, 327, 327, 312, 327, 322, 326, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664285191Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[2]R generation[0..12]: [189, 189, 189, 188, 189, 189, 183, 189, 186, 188, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.66429825Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664309228Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664318647Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664328416Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664337775Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664347073Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"":"downstairs","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664356662Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664365341Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"0","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.66437504Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"1","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664384599Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"client":"2","":"downstairs","session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664394807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664404146Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"f4e8bbd5-a2f8-4d88-a967-d3f2a0fd2c73 is now active with session: c5870fdf-ec01-4088-909d-8b6614027fdd","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664414035Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664422644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"session_id":"c5870fdf-ec01-4088-909d-8b6614027fdd"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664431753Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Activated sub_volume f4e8bbd5-a2f8-4d88-a967-d3f2a0fd2c73","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664439772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.66444952Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.664727965Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"task":"crutest"} 01/30 Write block 473 len 20480 data: 9 10 9 9 11 02/30 Read block 8 len 20480 03/30 Read block 103 len 24576 04/30 Write block 443 len 24576 data: 11 13 16 19 17 15 05/30 Write block 495 len 28672 data: 12 11 9 9 10 9 10 06/30 Write block 437 len 24576 data: 7 8 8 9 9 9 07/30 Write block 567 len 12288 data: 11 12 14 08/30 Flush 09/30 Write block 59 len 8192 data: 17 18 10/30 Read block 219 len 20480 11/30 Flush 12/30 Write block 531 len 40960 data: 14 17 18 17 17 17 14 16 21 22 13/30 Write block 546 len 20480 data: 15 16 14 14 15 14/30 Read block 155 len 36864 15/30 Write block 17 len 40960 data: 13 11 8 8 6 6 5 6 7 7 16/30 Read block 90 len 4096 17/30 Write block 348 len 20480 data: 19 14 13 13 16 18/30 Write block 240 len 32768 data: 11 9 11 9 12 16 15 18 19/30 Write block 471 len 40960 data: 7 8 10 11 10 10 12 9 12 12 20/30 Write block 159 len 24576 data: 14 13 14 12 15 14 21/30 Write block 114 len 4096 data: 8 22/30 Write block 489 len 40960 data: 8 8 10 13 16 12 13 12 10 10 23/30 Read block 497 len 28672 24/30 Flush 25/30 Write block 481 len 28672 data: 11 13 12 13 12 11 11 26/30 Write block 523 len 32768 data: 17 14 19 15 14 13 12 13 27/30 Write block 326 len 8192 data: 10 10 28/30 Write block 500 len 12288 data: 10 11 13 29/30 Read block 317 len 16384 30/30 Write block 461 len 36864 data: 18 20 23 16 17 15 12 12 8 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 22 24 Done 21 21 0 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:40.67304857Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:40.673234756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1990,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 189 330 330 327 F F F <--- 1 020-039 190 190 189 330 330 327 F F F <--- 2 040-059 190 190 189 329 329 327 F F F <--- 3 060-079 190 190 188 329 329 326 F F F <--- 4 080-099 189 189 189 327 327 327 F F F 5 100-119 190 190 189 330 330 327 F F F <--- 6 120-139 183 183 183 312 312 312 F F F 7 140-159 190 190 189 330 330 327 F F F <--- 8 160-179 190 190 186 330 330 322 F F F <--- 9 180-199 188 188 188 326 326 326 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 190 190 189 330 330 327 F F F <--- 13 260-279 189 189 189 327 327 327 F F F 14 280-299 188 188 188 326 326 326 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 179 179 179 306 306 306 T T F <--- 17 340-359 190 190 187 330 330 325 F F F <--- 18 360-379 187 187 187 325 325 325 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 190 190 190 328 328 328 F F F 22 440-459 190 190 190 328 328 328 F F F 23 460-479 190 190 190 330 330 328 T T F <--- 24 480-499 190 190 190 330 330 328 T T F <--- 25 500-519 190 190 190 328 328 328 T T F <--- 26 520-539 190 190 182 330 330 311 T T F <--- 27 540-559 190 190 189 330 330 327 F F F <--- 28 560-579 190 190 190 328 328 328 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 190, Max flush: 330 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 330 330 330 F F F 1 020-039 190 190 190 330 330 330 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 190 190 190 329 329 329 F F F 4 080-099 189 189 189 327 327 327 F F F 5 100-119 190 190 190 330 330 330 F F F 6 120-139 183 183 183 312 312 312 F F F 7 140-159 190 190 190 330 330 330 F F F 8 160-179 190 190 190 330 330 330 F F F 9 180-199 188 188 188 326 326 326 F F F 10 200-219 186 186 186 321 321 321 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 190 190 190 330 330 330 F F F 13 260-279 189 189 189 327 327 327 F F F 14 280-299 188 188 188 326 326 326 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 191 191 191 331 331 331 F F F 17 340-359 190 190 190 330 330 330 F F F 18 360-379 187 187 187 325 325 325 F F F 19 380-399 184 184 184 313 313 313 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 190 190 190 328 328 328 F F F 22 440-459 190 190 190 328 328 328 F F F 23 460-479 191 191 191 331 331 331 F F F 24 480-499 191 191 191 331 331 331 F F F 25 500-519 191 191 191 331 331 331 F F F 26 520-539 191 191 191 331 331 331 F F F 27 540-559 190 190 190 330 330 330 F F F 28 560-579 190 190 190 328 328 328 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 191, Max flush: 331 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.934895335Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.942903389Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943220459Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943244936Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943253585Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943265063Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Crucible c75ad71c-f782-4754-bfce-9c9b3e568b37 has session id: 44370cea-19fb-4de8-a21b-6dee453f407f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.94360866Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Upstairs opts: Upstairs UUID: c75ad71c-f782-4754-bfce-9c9b3e568b37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943671802Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Crucible stats registered with UUID: c75ad71c-f782-4754-bfce-9c9b3e568b37","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.94368374Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:54.943693109Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943833121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"0","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943849329Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"2","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.943861468Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"1","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944019258Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944035836Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"c75ad71c-f782-4754-bfce-9c9b3e568b37 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944069401Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.94407867Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"0","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944088469Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"1","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944098038Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"2","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944107497Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"2","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944117285Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"0","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944127094Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"1","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52602)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944176718Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"2","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35509)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944191356Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"1","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35560)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944202954Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"io task","client":"0","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.94494478Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"0","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.944959579Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"2","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945017641Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"1","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945351969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[0]R flush_numbers[0..12]: [330, 330, 329, 329, 327, 330, 312, 330, 330, 326, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945385295Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 190, 189, 190, 183, 190, 190, 188, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945396443Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945407322Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[1]R flush_numbers[0..12]: [330, 330, 329, 329, 327, 330, 312, 330, 330, 326, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.94541791Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 190, 189, 190, 183, 190, 190, 188, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945428299Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945439168Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[2]R flush_numbers[0..12]: [330, 330, 329, 329, 327, 330, 312, 330, 330, 326, 321, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945449586Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 190, 189, 190, 183, 190, 190, 188, 186, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945460005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945470634Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945480033Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945489621Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.94549879Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945508169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"":"downstairs","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945519048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945527826Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"0","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945537445Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"1","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945546994Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"client":"2","":"downstairs","session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945556883Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945566242Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"c75ad71c-f782-4754-bfce-9c9b3e568b37 is now active with session: 44370cea-19fb-4de8-a21b-6dee453f407f","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.9455762Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945585249Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"session_id":"44370cea-19fb-4de8-a21b-6dee453f407f"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945594288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Activated sub_volume c75ad71c-f782-4754-bfce-9c9b3e568b37","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945602167Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945610306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.945911028Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"task":"crutest"} 01/30 Write block 280 len 36864 data: 18 18 18 19 15 17 20 17 13 02/30 Write block 116 len 20480 data: 12 10 12 11 12 03/30 Read block 297 len 28672 04/30 Write block 430 len 12288 data: 14 15 15 05/30 Write block 546 len 4096 data: 16 06/30 Flush 07/30 Write block 119 len 12288 data: 12 13 12 08/30 Read block 293 len 16384 09/30 Write block 437 len 28672 data: 8 9 9 10 10 10 12 10/30 Write block 252 len 32768 data: 15 13 12 13 16 12 14 13 11/30 Write block 391 len 12288 data: 16 17 15 12/30 Read block 527 len 20480 13/30 Write block 148 len 4096 data: 23 14/30 Write block 546 len 4096 data: 17 15/30 Read block 586 len 28672 16/30 Write block 359 len 36864 data: 14 14 16 16 17 15 16 16 14 17/30 Read block 587 len 32768 18/30 Read block 122 len 40960 19/30 Write block 349 len 28672 data: 15 14 14 17 14 13 12 20/30 Read block 79 len 8192 21/30 Write block 201 len 4096 data: 10 22/30 Read block 298 len 24576 23/30 Read block 462 len 20480 24/30 Read block 132 len 24576 25/30 Read block 228 len 36864 26/30 Write block 425 len 32768 data: 18 17 17 15 17 15 16 16 27/30 Write block 191 len 32768 data: 12 15 17 14 13 11 9 9 28/30 Read block 41 len 36864 29/30 Read block 61 len 4096 30/30 Write block 436 len 20480 data: 11 9 10 10 11 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 29 5 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) None Some(JobId(1005)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:07:54.954621933Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:07:54.954795031Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":1999,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 330 330 330 F F F 1 020-039 190 190 190 330 330 330 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 190 190 190 329 329 329 F F F 4 080-099 189 189 189 327 327 327 F F F 5 100-119 192 190 192 332 330 332 T T T <--- 6 120-139 192 183 192 332 312 332 T T T <--- 7 140-159 190 190 190 330 330 330 T F T <--- 8 160-179 190 190 190 330 330 330 F F F 9 180-199 188 188 188 326 326 326 T F T <--- 10 200-219 186 186 186 321 321 321 T F T <--- 11 220-239 185 185 185 319 319 319 F F F 12 240-259 190 190 190 330 330 330 T F T <--- 13 260-279 189 189 189 327 327 327 F F F 14 280-299 192 188 192 332 326 332 F T F <--- 15 300-319 188 188 188 326 326 326 F F F 16 320-339 191 191 191 331 331 331 F F F 17 340-359 190 190 190 330 330 330 T F T <--- 18 360-379 187 187 187 325 325 325 T F T <--- 19 380-399 184 184 184 313 313 313 T F T <--- 20 400-419 180 180 180 309 309 309 F F F 21 420-439 192 190 192 332 328 332 T T T <--- 22 440-459 190 190 190 328 328 328 T F T <--- 23 460-479 191 191 191 331 331 331 F F F 24 480-499 191 191 191 331 331 331 F F F 25 500-519 191 191 191 331 331 331 F F F 26 520-539 191 191 191 331 331 331 F F F 27 540-559 192 190 192 332 330 332 T T T <--- 28 560-579 190 190 190 328 328 328 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 192, Max flush: 332 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 330 330 330 F F F 1 020-039 190 190 190 330 330 330 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 190 190 190 329 329 329 F F F 4 080-099 189 189 189 327 327 327 F F F 5 100-119 193 193 193 333 333 333 F F F 6 120-139 193 193 193 333 333 333 F F F 7 140-159 193 193 193 333 333 333 F F F 8 160-179 190 190 190 330 330 330 F F F 9 180-199 193 193 193 333 333 333 F F F 10 200-219 193 193 193 333 333 333 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 193 193 193 333 333 333 F F F 13 260-279 189 189 189 327 327 327 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 191 191 191 331 331 331 F F F 17 340-359 193 193 193 333 333 333 F F F 18 360-379 193 193 193 333 333 333 F F F 19 380-399 193 193 193 333 333 333 F F F 20 400-419 180 180 180 309 309 309 F F F 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 191 191 191 331 331 331 F F F 24 480-499 191 191 191 331 331 331 F F F 25 500-519 191 191 191 331 331 331 F F F 26 520-539 191 191 191 331 331 331 F F F 27 540-559 193 193 193 333 333 333 F F F 28 560-579 190 190 190 328 328 328 F F F 29 580-599 183 183 183 312 312 312 F F F Max gen: 193, Max flush: 333 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.217551881Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:09.224554193Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.224608666Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.2314409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.231696507Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.231733593Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.231743002Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.23175458Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Crucible 50d65ffb-4e75-40a5-ba17-399ce8cd4d9b has session id: 94ad154f-dfe6-4bc2-bf41-be2d969006f6","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232091107Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Upstairs opts: Upstairs UUID: 50d65ffb-4e75-40a5-ba17-399ce8cd4d9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.23215338Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Crucible stats registered with UUID: 50d65ffb-4e75-40a5-ba17-399ce8cd4d9b","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232163658Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:09.232172577Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.2323105Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"2","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232327048Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"1","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232336976Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"0","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232496476Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232519963Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"50d65ffb-4e75-40a5-ba17-399ce8cd4d9b active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232533461Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.23254211Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"0","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232553549Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"1","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232563058Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"2","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232572536Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"2","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232582375Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"0","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232592254Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"1","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53632)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232677703Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"1","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39936)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.23270288Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"0","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59686)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.232714118Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"io task","client":"2","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233323121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"1","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233348508Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"0","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233360136Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"2","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233688645Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[0]R flush_numbers[0..12]: [330, 330, 329, 329, 327, 333, 333, 333, 330, 333, 333, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233708772Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 190, 189, 193, 193, 193, 190, 193, 193, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233719281Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.23373005Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[1]R flush_numbers[0..12]: [330, 330, 329, 329, 327, 333, 333, 333, 330, 333, 333, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233743068Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 190, 189, 193, 193, 193, 190, 193, 193, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233753607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233764515Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[2]R flush_numbers[0..12]: [330, 330, 329, 329, 327, 333, 333, 333, 330, 333, 333, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233774954Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 190, 189, 193, 193, 193, 190, 193, 193, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233785373Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233796121Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.2338054Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233815379Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233824758Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233834096Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"":"downstairs","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233843875Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233852674Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"0","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233862533Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"1","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233872332Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"client":"2","":"downstairs","session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.23388376Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233893089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"50d65ffb-4e75-40a5-ba17-399ce8cd4d9b is now active with session: 94ad154f-dfe6-4bc2-bf41-be2d969006f6","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233902948Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233911627Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"session_id":"94ad154f-dfe6-4bc2-bf41-be2d969006f6"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233920665Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Activated sub_volume 50d65ffb-4e75-40a5-ba17-399ce8cd4d9b","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233928644Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.233936883Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.234253723Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} 01/30 Write block 330 len 4096 data: 11 02/30 Read block 388 len 28672 03/30 Write block 8 len 12288 data: 8 9 13 04/30 Write block 250 len 8192 data: 16 15 05/30 Read block 249 len 16384 06/30 Write block 371 len 24576 data: 13 14 15 15 17 18 07/30 Write block 109 len 20480 data: 17 16 15 10 8 08/30 Read block 418 len 16384 09/30 Write block 522 len 8192 data: 16 18 10/30 Read block 168 len 32768 11/30 Write block 63 len 40960 data: 16 15 15 13 12 11 11 11 16 17 12/30 Flush 13/30 Write block 553 len 16384 data: 12 8 11 10 14/30 Write block 366 len 32768 data: 17 15 12 12 15 14 15 16 15/30 Write block 96 len 4096 data: 16 16/30 Read block 330 len 24576 17/30 Write block 162 len 16384 data: 13 16 15 10 18/30 Read block 513 len 8192 19/30 Read block 209 len 24576 20/30 Read block 276 len 8192 21/30 Write block 245 len 32768 data: 17 16 19 20 21 17 16 16 22/30 Write block 328 len 24576 data: 10 11 12 6 8 9 23/30 Write block 492 len 16384 data: 14 17 13 14 24/30 Read block 462 len 12288 25/30 Write block 588 len 16384 data: 10 10 14 15 26/30 Read block 180 len 24576 27/30 Read block 115 len 32768 28/30 Write block 578 len 28672 data: 15 14 15 16 16 16 13 29/30 Write block 403 len 20480 data: 16 15 15 13 14 30/30 Read block 151 len 16384 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:09.242364724Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:09.242543652Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2008,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 194 194 330 334 334 F F F <--- 1 020-039 190 190 190 330 330 330 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 190 194 194 329 334 334 F F F <--- 4 080-099 189 189 189 327 327 327 F T T <--- 5 100-119 193 194 194 333 334 334 F F F <--- 6 120-139 193 193 193 333 333 333 F F F 7 140-159 193 193 193 333 333 333 F F F 8 160-179 190 190 190 330 330 330 F T T <--- 9 180-199 193 193 193 333 333 333 F F F 10 200-219 193 193 193 333 333 333 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 193 194 194 333 334 334 F T T <--- 13 260-279 189 189 189 327 327 327 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 191 194 194 331 334 334 T T T <--- 17 340-359 193 193 193 333 333 333 F F F 18 360-379 193 194 194 333 334 334 F T T <--- 19 380-399 193 193 193 333 333 333 F F F 20 400-419 180 180 180 309 309 309 F T T <--- 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 191 191 191 331 331 331 F F F 24 480-499 191 191 191 331 331 331 F T T <--- 25 500-519 191 191 191 331 331 331 F F F 26 520-539 191 194 194 331 334 334 F F F <--- 27 540-559 193 193 193 333 333 333 F T T <--- 28 560-579 190 190 190 328 328 328 F T T <--- 29 580-599 183 183 183 312 312 312 F T T <--- Max gen: 194, Max flush: 334 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 334 334 334 F F F 1 020-039 190 190 190 330 330 330 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 195 195 195 335 335 335 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 193 193 193 333 333 333 F F F 7 140-159 193 193 193 333 333 333 F F F 8 160-179 195 195 195 335 335 335 F F F 9 180-199 193 193 193 333 333 333 F F F 10 200-219 193 193 193 333 333 333 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 195 195 195 335 335 335 F F F 13 260-279 189 189 189 327 327 327 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 195 195 195 335 335 335 F F F 17 340-359 193 193 193 333 333 333 F F F 18 360-379 195 195 195 335 335 335 F F F 19 380-399 193 193 193 333 333 333 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 191 191 191 331 331 331 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 191 191 191 331 331 331 F F F 26 520-539 194 194 194 334 334 334 F F F 27 540-559 195 195 195 335 335 335 F F F 28 560-579 195 195 195 335 335 335 F F F 29 580-599 195 195 195 335 335 335 F F F Max gen: 195, Max flush: 335 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.506298526Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:23.513329024Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.513379188Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520095526Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520354693Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52037795Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520386529Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520398067Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Crucible 8eba5c91-7343-4cd2-99d2-9a7a85479f84 has session id: 25a193d7-5e93-49e2-a13f-cdd29331b7e5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52077163Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Upstairs opts: Upstairs UUID: 8eba5c91-7343-4cd2-99d2-9a7a85479f84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520801056Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Crucible stats registered with UUID: 8eba5c91-7343-4cd2-99d2-9a7a85479f84","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520812355Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:23.520821794Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520951367Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"1","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520969675Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"0","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.520979654Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"2","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521116516Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521140613Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"8eba5c91-7343-4cd2-99d2-9a7a85479f84 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521187807Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521197496Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"0","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521207715Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"1","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521217324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"2","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521226752Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"2","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521236651Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"0","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52124855Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"1","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:52638)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52132409Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"0","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58665)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521426197Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"2","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35850)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521446684Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"io task","client":"1","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.521974868Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"1","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522019502Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"2","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522046938Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"0","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522316824Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[0]R flush_numbers[0..12]: [334, 330, 329, 334, 335, 334, 333, 333, 335, 333, 333, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522336712Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[0]R generation[0..12]: [194, 190, 190, 194, 195, 194, 193, 193, 195, 193, 193, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52234744Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522358479Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[1]R flush_numbers[0..12]: [334, 330, 329, 334, 335, 334, 333, 333, 335, 333, 333, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522369157Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[1]R generation[0..12]: [194, 190, 190, 194, 195, 194, 193, 193, 195, 193, 193, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522379756Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522390655Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[2]R flush_numbers[0..12]: [334, 330, 329, 334, 335, 334, 333, 333, 335, 333, 333, 319]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522403763Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[2]R generation[0..12]: [194, 190, 190, 194, 195, 194, 193, 193, 195, 193, 193, 185]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522414162Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52242486Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522434179Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522443978Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522453257Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522462596Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"":"downstairs","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522472194Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522480903Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"0","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522490422Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"1","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522500001Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"client":"2","":"downstairs","session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.52251Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522519388Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"8eba5c91-7343-4cd2-99d2-9a7a85479f84 is now active with session: 25a193d7-5e93-49e2-a13f-cdd29331b7e5","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522529327Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522537896Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"session_id":"25a193d7-5e93-49e2-a13f-cdd29331b7e5"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522548265Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Activated sub_volume 8eba5c91-7343-4cd2-99d2-9a7a85479f84","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-10-01T01:08:23.522556184Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522569452Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.522811571Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} 01/30 Write block 15 len 28672 data: 15 13 14 12 9 9 7 02/30 Write block 593 len 16384 data: 12 12 13 11 03/30 Flush 04/30 Write block 475 len 12288 data: 11 11 13 05/30 Read block 422 len 40960 06/30 Write block 287 len 8192 data: 18 14 07/30 Read block 529 len 20480 08/30 Write block 222 len 12288 data: 18 13 13 09/30 Read block 328 len 36864 10/30 Write block 592 len 4096 data: 13 11/30 Flush 12/30 Write block 185 len 4096 data: 16 13/30 Read block 442 len 8192 14/30 Read block 102 len 40960 15/30 Read block 130 len 32768 16/30 Read block 256 len 16384 17/30 Write block 525 len 8192 data: 20 16 18/30 Write block 473 len 12288 data: 11 12 12 19/30 Read block 144 len 32768 20/30 Flush 21/30 Write block 332 len 4096 data: 9 22/30 Read block 137 len 28672 23/30 Write block 389 len 16384 data: 13 17 17 18 24/30 Read block 538 len 8192 25/30 Write block 126 len 36864 data: 23 21 17 21 20 23 25 20 20 26/30 Read block 581 len 36864 27/30 Read block 392 len 8192 28/30 Write block 178 len 8192 data: 13 14 29/30 Read block 379 len 12288 30/30 Read block 107 len 20480 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 3 Done Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 0 27 Done 1 26 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:23.531160303Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:23.531349189Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2017,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 336 336 336 F F F 1 020-039 196 196 196 336 336 336 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 195 195 195 335 335 335 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 193 193 193 333 333 333 F T T <--- 7 140-159 193 193 193 333 333 333 F F F 8 160-179 195 195 195 335 335 335 F T T <--- 9 180-199 193 196 196 333 338 338 F F F <--- 10 200-219 193 193 193 333 333 333 F F F 11 220-239 185 196 196 319 337 337 F F F <--- 12 240-259 195 195 195 335 335 335 F F F 13 260-279 189 189 189 327 327 327 F F F 14 280-299 192 196 196 332 337 337 F F F <--- 15 300-319 188 188 188 326 326 326 F F F 16 320-339 195 195 195 335 335 335 F T T <--- 17 340-359 193 193 193 333 333 333 F F F 18 360-379 195 195 195 335 335 335 F F F 19 380-399 193 193 193 333 333 333 F T T <--- 20 400-419 195 195 195 335 335 335 F F F 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 191 196 196 331 338 338 T F F <--- 24 480-499 195 195 195 335 335 335 F F F 25 500-519 191 191 191 331 331 331 F F F 26 520-539 194 196 196 334 338 338 F F F <--- 27 540-559 195 195 195 335 335 335 F F F 28 560-579 195 195 195 335 335 335 F F F 29 580-599 196 196 196 336 337 337 F F F <--- Max gen: 196, Max flush: 338 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 336 336 336 F F F 1 020-039 196 196 196 336 336 336 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 195 195 195 335 335 335 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 197 197 197 339 339 339 F F F 7 140-159 193 193 193 333 333 333 F F F 8 160-179 197 197 197 339 339 339 F F F 9 180-199 196 196 196 338 338 338 F F F 10 200-219 193 193 193 333 333 333 F F F 11 220-239 196 196 196 337 337 337 F F F 12 240-259 195 195 195 335 335 335 F F F 13 260-279 189 189 189 327 327 327 F F F 14 280-299 196 196 196 337 337 337 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 197 197 197 339 339 339 F F F 17 340-359 193 193 193 333 333 333 F F F 18 360-379 195 195 195 335 335 335 F F F 19 380-399 197 197 197 339 339 339 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 196 196 196 338 338 338 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 191 191 191 331 331 331 F F F 26 520-539 196 196 196 338 338 338 F F F 27 540-559 195 195 195 335 335 335 F F F 28 560-579 195 195 195 335 335 335 F F F 29 580-599 196 196 196 337 337 337 F F F Max gen: 197, Max flush: 339 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.786463328Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.794389663Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.794689325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80fc9ce295b95c254ddb0b25aafc73ba9baf1b9c\",\n git_commit_timestamp: \"2026-10-01T00:29:57.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.794713412Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.794721781Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.794733169Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Crucible f06bd7c8-0675-4a36-aee5-41aa233b4bdc has session id: 626d8ebd-0368-4743-9287-426c8a07dc25","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795095283Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Upstairs opts: Upstairs UUID: f06bd7c8-0675-4a36-aee5-41aa233b4bdc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795127089Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Crucible stats registered with UUID: f06bd7c8-0675-4a36-aee5-41aa233b4bdc","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795138278Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:37.795147607Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795313656Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"1","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795331143Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"0","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795343382Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"2","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795465046Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795482324Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"f06bd7c8-0675-4a36-aee5-41aa233b4bdc active request set","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795496892Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795505681Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"0","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.79551526Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"1","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795524969Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"2","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795534648Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"2","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795544636Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"0","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795554455Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"1","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"ds_connection connected from Ok(127.0.0.1:54827)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795627486Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"1","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"ds_connection connected from Ok(127.0.0.1:39818)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795652583Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"0","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"ds_connection connected from Ok(127.0.0.1:48519)","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.795722774Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"io task","client":"2","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796346275Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"0","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796363742Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"1","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796376921Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"2","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796748624Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[0]R flush_numbers[0..12]: [336, 336, 329, 334, 335, 334, 339, 333, 339, 338, 333, 337]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796782539Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[0]R generation[0..12]: [196, 196, 190, 194, 195, 194, 197, 193, 197, 196, 193, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796793738Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796804757Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[1]R flush_numbers[0..12]: [336, 336, 329, 334, 335, 334, 339, 333, 339, 338, 333, 337]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796815325Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[1]R generation[0..12]: [196, 196, 190, 194, 195, 194, 197, 193, 197, 196, 193, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796825864Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796836782Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[2]R flush_numbers[0..12]: [336, 336, 329, 334, 335, 334, 339, 333, 339, 338, 333, 337]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796847421Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[2]R generation[0..12]: [196, 196, 190, 194, 195, 194, 197, 193, 197, 196, 193, 196]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.79685786Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796869138Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796878547Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796888306Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796897615Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796908463Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"":"downstairs","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796918442Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796927401Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"0","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.79693736Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"1","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796946998Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"client":"2","":"downstairs","session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796956697Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796965716Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"f06bd7c8-0675-4a36-aee5-41aa233b4bdc is now active with session: 626d8ebd-0368-4743-9287-426c8a07dc25","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796975505Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796984124Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"session_id":"626d8ebd-0368-4743-9287-426c8a07dc25"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.796993093Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Activated sub_volume f06bd7c8-0675-4a36-aee5-41aa233b4bdc","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.797001192Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.797009421Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.797259609Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"task":"crutest"} 01/30 Write block 574 len 24576 data: 11 13 15 14 16 15 02/30 Write block 103 len 16384 data: 15 14 14 13 03/30 Write block 280 len 12288 data: 19 19 19 04/30 Read block 35 len 24576 05/30 Read block 211 len 20480 06/30 Write block 343 len 32768 data: 16 16 18 20 21 20 16 15 07/30 Write block 209 len 40960 data: 24 22 20 23 21 21 18 17 16 19 08/30 Read block 215 len 8192 09/30 Write block 505 len 4096 data: 23 10/30 Read block 168 len 4096 11/30 Read block 61 len 12288 12/30 Read block 209 len 40960 13/30 Flush 14/30 Write block 287 len 20480 data: 19 15 12 8 9 15/30 Read block 312 len 28672 16/30 Write block 326 len 4096 data: 11 17/30 Flush 18/30 Write block 80 len 8192 data: 16 13 19/30 Write block 206 len 40960 data: 20 22 23 25 23 21 24 22 22 19 20/30 Read block 247 len 40960 21/30 Read block 457 len 32768 22/30 Read block 239 len 12288 23/30 Read block 329 len 40960 24/30 Write block 184 len 8192 data: 17 17 25/30 Read block 443 len 4096 26/30 Read block 95 len 36864 27/30 Write block 83 len 36864 data: 11 15 16 16 16 22 19 16 14 28/30 Write block 261 len 4096 data: 12 29/30 Read block 224 len 8192 30/30 Write block 584 len 16384 data: 14 10 9 9 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 3 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-01T01:08:37.805326456Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-10-01T01:08:37.805538999Z","hostname":"w-01M3TENQCF8Q6JV57EJ5FJRMYJ","pid":2026,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 336 336 336 F F F 1 020-039 196 196 196 336 336 336 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 195 195 195 335 335 335 T T F <--- 5 100-119 198 198 194 340 340 334 F F T <--- 6 120-139 197 197 197 339 339 339 F F F 7 140-159 193 193 193 333 333 333 F F F 8 160-179 197 197 197 339 339 339 F F F 9 180-199 196 196 196 338 338 338 T T F <--- 10 200-219 198 198 193 340 340 333 T T F <--- 11 220-239 196 196 196 337 337 337 F F F 12 240-259 195 195 195 335 335 335 F F F 13 260-279 189 189 189 327 327 327 T T F <--- 14 280-299 198 198 196 341 341 337 F F F <--- 15 300-319 188 188 188 326 326 326 F F F 16 320-339 198 198 197 341 341 339 F F F <--- 17 340-359 198 198 193 340 340 333 F F F <--- 18 360-379 195 195 195 335 335 335 F F F 19 380-399 197 197 197 339 339 339 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 196 196 196 338 338 338 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 198 198 191 340 340 331 F F F <--- 26 520-539 196 196 196 338 338 338 F F F 27 540-559 195 195 195 335 335 335 F F F 28 560-579 198 198 195 340 340 335 F F T <--- 29 580-599 196 196 196 337 337 337 T T F <--- Max gen: 198, Max flush: 341 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 336 336 336 F F F 1 020-039 196 196 196 336 336 336 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 194 194 194 334 334 334 F F F 4 080-099 199 199 199 342 342 342 F F F 5 100-119 198 198 198 340 340 340 F F F 6 120-139 197 197 197 339 339 339 F F F 7 140-159 193 193 193 333 333 333 F F F 8 160-179 197 197 197 339 339 339 F F F 9 180-199 199 199 199 342 342 342 F F F 10 200-219 199 199 199 342 342 342 F F F 11 220-239 196 196 196 337 337 337 F F F 12 240-259 195 195 195 335 335 335 F F F 13 260-279 199 199 199 342 342 342 F F F 14 280-299 198 198 198 341 341 341 F F F 15 300-319 188 188 188 326 326 326 F F F 16 320-339 198 198 198 341 341 341 F F F 17 340-359 198 198 198 340 340 340 F F F 18 360-379 195 195 195 335 335 335 F F F 19 380-399 197 197 197 339 339 339 F F F 20 400-419 195 195 195 335 335 335 F F F 21 420-439 193 193 193 333 333 333 F F F 22 440-459 193 193 193 333 333 333 F F F 23 460-479 196 196 196 338 338 338 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 198 198 198 340 340 340 F F F 26 520-539 196 196 196 338 338 338 F F F 27 540-559 195 195 195 335 335 335 F F F 28 560-579 198 198 198 340 340 340 F F F 29 580-599 199 199 199 342 342 342 F F F Max gen: 199, Max flush: 342 23:58 Test duration Test completed real 23:58.013715359 user 25.408738795 sys 14.711701409 trap 0.021929276 tflt 0.063466047 dflt 0.073327254 kflt 0.003986769 lock 33:46:26.181442029 slp 1:57:31.938291812 lat 2.712741603 stop 0.021332584